[ 466.461558] env[62506]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=62506) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 466.461901] env[62506]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=62506) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 466.462041] env[62506]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=62506) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 466.462384] env[62506]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 466.557927] env[62506]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=62506) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:349}} [ 466.568178] env[62506]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.010s {{(pid=62506) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:372}} [ 467.169632] env[62506]: INFO nova.virt.driver [None req-b6089ede-88b0-4f1c-b3f9-45d5da8fb39c None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 467.239384] env[62506]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 467.239525] env[62506]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 467.239621] env[62506]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=62506) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 470.491732] env[62506]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-c2e89267-3019-4ac3-b11e-4a3eef0d557b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.508170] env[62506]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=62506) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 470.508290] env[62506]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-1b3d57a2-fdde-49f1-bba6-4fdc580ca1d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.543383] env[62506]: INFO oslo_vmware.api [-] Successfully established new session; session ID is 814f9. [ 470.543538] env[62506]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.304s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 470.544101] env[62506]: INFO nova.virt.vmwareapi.driver [None req-b6089ede-88b0-4f1c-b3f9-45d5da8fb39c None None] VMware vCenter version: 7.0.3 [ 470.547562] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae69c8ed-5598-4a87-be2a-79e0811822bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.569791] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0f1b318-d7cf-4f7d-9624-ceb3622c1478 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.575903] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d59ea52c-285e-4a71-8540-340b441bf47e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.582721] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8511b814-9c55-4a8a-a6af-498fae583fb4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.596309] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd531349-1f45-4627-a6cc-1bf662bbaaa1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.602432] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb4d6a54-0da9-42e1-9b61-8a8a7fd4726a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.632482] env[62506]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-09424a5a-e0c4-4d31-a82f-cdb72777b28a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 470.638629] env[62506]: DEBUG nova.virt.vmwareapi.driver [None req-b6089ede-88b0-4f1c-b3f9-45d5da8fb39c None None] Extension org.openstack.compute already exists. {{(pid=62506) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:228}} [ 470.641357] env[62506]: INFO nova.compute.provider_config [None req-b6089ede-88b0-4f1c-b3f9-45d5da8fb39c None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 471.144771] env[62506]: DEBUG nova.context [None req-b6089ede-88b0-4f1c-b3f9-45d5da8fb39c None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),38e6d02b-9ee7-40d3-9d00-7cee5277a963(cell1) {{(pid=62506) load_cells /opt/stack/nova/nova/context.py:464}} [ 471.147713] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 471.147940] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 471.148909] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 471.149130] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Acquiring lock "38e6d02b-9ee7-40d3-9d00-7cee5277a963" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 471.149325] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Lock "38e6d02b-9ee7-40d3-9d00-7cee5277a963" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 471.150343] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Lock "38e6d02b-9ee7-40d3-9d00-7cee5277a963" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 471.170269] env[62506]: INFO dbcounter [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Registered counter for database nova_cell0 [ 471.178399] env[62506]: INFO dbcounter [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Registered counter for database nova_cell1 [ 471.181527] env[62506]: DEBUG oslo_db.sqlalchemy.engines [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=62506) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:395}} [ 471.181898] env[62506]: DEBUG oslo_db.sqlalchemy.engines [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=62506) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:395}} [ 471.186699] env[62506]: ERROR nova.db.main.api [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 471.186699] env[62506]: result = function(*args, **kwargs) [ 471.186699] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 471.186699] env[62506]: return func(*args, **kwargs) [ 471.186699] env[62506]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 471.186699] env[62506]: result = fn(*args, **kwargs) [ 471.186699] env[62506]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 471.186699] env[62506]: return f(*args, **kwargs) [ 471.186699] env[62506]: File "/opt/stack/nova/nova/objects/service.py", line 556, in _db_service_get_minimum_version [ 471.186699] env[62506]: return db.service_get_minimum_version(context, binaries) [ 471.186699] env[62506]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 471.186699] env[62506]: _check_db_access() [ 471.186699] env[62506]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 471.186699] env[62506]: stacktrace = ''.join(traceback.format_stack()) [ 471.186699] env[62506]: [ 471.187518] env[62506]: ERROR nova.db.main.api [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 471.187518] env[62506]: result = function(*args, **kwargs) [ 471.187518] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 471.187518] env[62506]: return func(*args, **kwargs) [ 471.187518] env[62506]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 471.187518] env[62506]: result = fn(*args, **kwargs) [ 471.187518] env[62506]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 471.187518] env[62506]: return f(*args, **kwargs) [ 471.187518] env[62506]: File "/opt/stack/nova/nova/objects/service.py", line 556, in _db_service_get_minimum_version [ 471.187518] env[62506]: return db.service_get_minimum_version(context, binaries) [ 471.187518] env[62506]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 471.187518] env[62506]: _check_db_access() [ 471.187518] env[62506]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 471.187518] env[62506]: stacktrace = ''.join(traceback.format_stack()) [ 471.187518] env[62506]: [ 471.188154] env[62506]: WARNING nova.objects.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Failed to get minimum service version for cell 38e6d02b-9ee7-40d3-9d00-7cee5277a963 [ 471.188154] env[62506]: WARNING nova.objects.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 471.188457] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Acquiring lock "singleton_lock" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 471.188616] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Acquired lock "singleton_lock" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 471.188853] env[62506]: DEBUG oslo_concurrency.lockutils [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Releasing lock "singleton_lock" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 471.189186] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Full set of CONF: {{(pid=62506) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 471.189338] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ******************************************************************************** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2804}} [ 471.189465] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] Configuration options gathered from: {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2805}} [ 471.189600] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2806}} [ 471.189794] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2807}} [ 471.189920] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ================================================================================ {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2809}} [ 471.190140] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] allow_resize_to_same_host = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.190314] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] arq_binding_timeout = 300 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.190446] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] backdoor_port = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.190575] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] backdoor_socket = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.190736] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] block_device_allocate_retries = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.190898] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] block_device_allocate_retries_interval = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.191083] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cert = self.pem {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.191257] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.191427] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute_monitors = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.191595] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] config_dir = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.191763] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] config_drive_format = iso9660 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.191895] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.192068] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] config_source = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.192241] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] console_host = devstack {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.192409] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] control_exchange = nova {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.192564] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cpu_allocation_ratio = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.192723] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] daemon = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.192890] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] debug = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.193052] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] default_access_ip_network_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.193221] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] default_availability_zone = nova {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.193379] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] default_ephemeral_format = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.193537] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] default_green_pool_size = 1000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.193772] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.193935] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] default_schedule_zone = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.194104] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] disk_allocation_ratio = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.194289] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] enable_new_services = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.194477] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] enabled_apis = ['osapi_compute'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.194645] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] enabled_ssl_apis = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.194806] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] flat_injected = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.194963] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] force_config_drive = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.195140] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] force_raw_images = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.195329] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] graceful_shutdown_timeout = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.195505] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] heal_instance_info_cache_interval = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.195724] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] host = cpu-1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.195903] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] initial_cpu_allocation_ratio = 4.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.196080] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] initial_disk_allocation_ratio = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.196243] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] initial_ram_allocation_ratio = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.196456] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.196621] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_build_timeout = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.196782] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_delete_interval = 300 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.196945] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_format = [instance: %(uuid)s] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.197126] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_name_template = instance-%08x {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.197291] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_usage_audit = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.197460] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_usage_audit_period = month {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.197626] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.197790] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] instances_path = /opt/stack/data/nova/instances {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.197958] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] internal_service_availability_zone = internal {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.198125] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] key = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.198290] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] live_migration_retry_count = 30 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.198465] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_color = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.198633] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_config_append = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.198802] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.198963] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_dir = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.199135] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.199268] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_options = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.199435] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_rotate_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.199603] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_rotate_interval_type = days {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.199768] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] log_rotation_type = none {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.199899] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.200049] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.200234] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.200410] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.200542] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.200707] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] long_rpc_timeout = 1800 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.200871] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] max_concurrent_builds = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.201043] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] max_concurrent_live_migrations = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.201208] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] max_concurrent_snapshots = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.201374] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] max_local_block_devices = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.201534] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] max_logfile_count = 30 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.201692] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] max_logfile_size_mb = 200 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.201853] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] maximum_instance_delete_attempts = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.202031] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metadata_listen = 0.0.0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.202208] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metadata_listen_port = 8775 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.202380] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metadata_workers = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.202543] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] migrate_max_retries = -1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.202712] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] mkisofs_cmd = genisoimage {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.202921] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] my_block_storage_ip = 10.180.1.21 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.203063] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] my_ip = 10.180.1.21 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.203275] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] my_shared_fs_storage_ip = 10.180.1.21 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.203446] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] network_allocate_retries = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.203628] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.203795] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] osapi_compute_listen = 0.0.0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.203962] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] osapi_compute_listen_port = 8774 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.204181] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] osapi_compute_unique_server_name_scope = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.204348] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] osapi_compute_workers = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.204568] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] password_length = 12 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.204738] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] periodic_enable = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.204907] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] periodic_fuzzy_delay = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.205088] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] pointer_model = usbtablet {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.205274] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] preallocate_images = none {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.205455] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] publish_errors = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.205586] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] pybasedir = /opt/stack/nova {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.205748] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ram_allocation_ratio = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.205909] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] rate_limit_burst = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.206094] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] rate_limit_except_level = CRITICAL {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.206260] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] rate_limit_interval = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.206420] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reboot_timeout = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.206583] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reclaim_instance_interval = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.206737] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] record = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.206902] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reimage_timeout_per_gb = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.207079] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] report_interval = 120 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.207246] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] rescue_timeout = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.207407] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reserved_host_cpus = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.207567] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reserved_host_disk_mb = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.207726] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reserved_host_memory_mb = 512 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.207946] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] reserved_huge_pages = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.208126] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] resize_confirm_window = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.208291] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] resize_fs_using_block_device = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.208458] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] resume_guests_state_on_host_boot = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.208648] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.208810] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] rpc_response_timeout = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.208971] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] run_external_periodic_tasks = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.209150] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] running_deleted_instance_action = reap {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.209316] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] running_deleted_instance_poll_interval = 1800 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.209474] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] running_deleted_instance_timeout = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.209632] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler_instance_sync_interval = 120 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.209799] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_down_time = 720 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.209965] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] servicegroup_driver = db {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.210134] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] shell_completion = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.210293] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] shelved_offload_time = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.210456] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] shelved_poll_interval = 3600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.210625] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] shutdown_timeout = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.210785] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] source_is_ipv6 = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.210944] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ssl_only = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.211217] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.211405] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] sync_power_state_interval = 600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.211572] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] sync_power_state_pool_size = 1000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.211741] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] syslog_log_facility = LOG_USER {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.211929] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] tempdir = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.212159] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] timeout_nbd = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.212345] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] transport_url = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.212509] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] update_resources_interval = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.212670] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] use_cow_images = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.212832] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] use_journal = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.212992] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] use_json = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.213167] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] use_rootwrap_daemon = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.213328] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] use_stderr = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.213488] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] use_syslog = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.213642] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vcpu_pin_set = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.213810] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plugging_is_fatal = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.213995] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plugging_timeout = 300 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.214203] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] virt_mkfs = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.214379] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] volume_usage_poll_interval = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.214546] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] watch_log_file = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.214715] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] web = /usr/share/spice-html5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2817}} [ 471.214902] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_brick.lock_path = /opt/stack/data/n-cpu-1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.215082] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_brick.wait_mpath_device_attempts = 4 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.215268] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_brick.wait_mpath_device_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.215466] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_concurrency.disable_process_locking = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.216383] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.216587] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.216768] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.216944] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_metrics.metrics_process_name = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.217139] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.217317] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.217507] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.auth_strategy = keystone {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.217677] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.compute_link_prefix = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.217856] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.218046] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.dhcp_domain = novalocal {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.218226] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.enable_instance_password = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.218398] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.glance_link_prefix = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.218565] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.218740] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.instance_list_cells_batch_strategy = distributed {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.218906] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.instance_list_per_project_cells = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.219083] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.list_records_by_skipping_down_cells = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.219256] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.local_metadata_per_cell = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.219432] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.max_limit = 1000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.219605] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.metadata_cache_expiration = 15 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.219780] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.neutron_default_tenant_id = default {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.219954] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.response_validation = warn {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.220142] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.use_neutron_default_nets = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.220317] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.220486] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_dynamic_failure_fatal = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.220652] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.220827] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_dynamic_ssl_certfile = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.221009] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_dynamic_targets = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.221181] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_jsonfile_path = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.221369] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api.vendordata_providers = ['StaticJSON'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.221568] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.backend = dogpile.cache.memcached {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.221739] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.backend_argument = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.221902] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.backend_expiration_time = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.222086] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.config_prefix = cache.oslo {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.222264] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.dead_timeout = 60.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.222431] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.debug_cache_backend = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.222597] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.enable_retry_client = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.222760] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.enable_socket_keepalive = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.222932] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.enabled = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.223112] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.enforce_fips_mode = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.223285] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.expiration_time = 600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.223451] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.hashclient_retry_attempts = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.223623] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.hashclient_retry_delay = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.223788] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_dead_retry = 300 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.223954] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_password = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.224137] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.224332] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.224505] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_pool_maxsize = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.224673] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_pool_unused_timeout = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.224839] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_sasl_enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.225032] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_servers = ['localhost:11211'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.225214] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_socket_timeout = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.225407] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.memcache_username = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.225582] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.proxies = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.225753] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_db = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.225915] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_password = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.226103] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_sentinel_service_name = mymaster {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.226289] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.226467] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_server = localhost:6379 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.226637] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_socket_timeout = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.226800] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.redis_username = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.226968] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.retry_attempts = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.227152] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.retry_delay = 0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.227326] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.socket_keepalive_count = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.227492] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.socket_keepalive_idle = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.227657] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.socket_keepalive_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.227818] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.tls_allowed_ciphers = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.227979] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.tls_cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.228153] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.tls_certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.228322] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.tls_enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.228483] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cache.tls_keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.228655] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.228834] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.auth_type = password {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.228998] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.229193] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.catalog_info = volumev3::publicURL {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.229361] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.229528] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.229693] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.cross_az_attach = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.229857] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.debug = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.230022] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.endpoint_template = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.230190] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.http_retries = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.230357] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.230516] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.230687] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.os_region_name = RegionOne {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.230849] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.231027] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cinder.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.231205] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.231369] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.cpu_dedicated_set = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.231531] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.cpu_shared_set = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.231701] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.image_type_exclude_list = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.231867] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.live_migration_wait_for_vif_plug = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.232050] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.max_concurrent_disk_ops = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.232225] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.max_disk_devices_to_attach = -1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.232392] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.232567] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.232736] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.resource_provider_association_refresh = 300 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.232902] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.sharing_providers_max_uuids_per_request = 200 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.233078] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.shutdown_retry_interval = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.233266] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.233451] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] conductor.workers = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.233632] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] console.allowed_origins = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.233793] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] console.ssl_ciphers = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.233963] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] console.ssl_minimum_version = default {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.234153] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] consoleauth.enforce_session_timeout = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.234353] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] consoleauth.token_ttl = 600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.234535] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.234697] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.234866] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.235039] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.235204] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.235378] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.235545] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.235704] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.235868] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.236038] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.236205] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.236365] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.236542] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.236728] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.service_type = accelerator {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.236893] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.237064] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.237229] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.237392] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.237574] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.237739] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] cyborg.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.237914] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.asyncio_connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.238089] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.asyncio_slave_connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.238265] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.backend = sqlalchemy {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.238443] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.238614] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.connection_debug = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.238787] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.connection_parameters = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.238953] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.connection_recycle_time = 3600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.239136] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.connection_trace = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.239305] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.db_inc_retry_interval = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.239474] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.db_max_retries = 20 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.239639] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.db_max_retry_interval = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.239804] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.db_retry_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.239969] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.max_overflow = 50 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.240148] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.max_pool_size = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.240312] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.max_retries = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.240487] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.mysql_sql_mode = TRADITIONAL {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.240650] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.mysql_wsrep_sync_wait = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.240809] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.pool_timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.240973] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.retry_interval = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.241148] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.slave_connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.241312] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.sqlite_synchronous = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.241479] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] database.use_db_reconnect = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.241650] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.asyncio_connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.241808] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.asyncio_slave_connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.241977] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.backend = sqlalchemy {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.242161] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.242341] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.connection_debug = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.242518] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.connection_parameters = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.242681] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.connection_recycle_time = 3600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.242846] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.connection_trace = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.243025] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.db_inc_retry_interval = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.243189] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.db_max_retries = 20 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.243355] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.db_max_retry_interval = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.243520] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.db_retry_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.243685] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.max_overflow = 50 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.243848] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.max_pool_size = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.244017] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.max_retries = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.244236] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.244411] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.mysql_wsrep_sync_wait = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.244577] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.pool_timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.244742] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.retry_interval = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.244903] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.slave_connection = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.245079] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] api_database.sqlite_synchronous = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.245262] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] devices.enabled_mdev_types = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.245444] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.245618] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ephemeral_storage_encryption.default_format = luks {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.245787] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ephemeral_storage_encryption.enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.245953] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ephemeral_storage_encryption.key_size = 512 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.246138] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.api_servers = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.246307] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.246499] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.246669] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.246833] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.246994] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.247175] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.debug = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.247348] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.default_trusted_certificate_ids = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.247515] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.enable_certificate_validation = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.247678] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.enable_rbd_download = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.247840] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.248014] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.248190] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.248354] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.248516] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.248682] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.num_retries = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.248854] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.rbd_ceph_conf = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.249029] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.rbd_connect_timeout = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.249206] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.rbd_pool = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.249377] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.rbd_user = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.249544] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.249704] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.249866] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.250047] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.service_type = image {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.250217] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.250380] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.250540] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.250700] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.250882] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.251060] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.verify_glance_signatures = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.251230] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] glance.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.251400] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] guestfs.debug = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.251569] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.251733] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.auth_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.251894] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.252065] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.252242] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.252401] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.252562] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.252721] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.252883] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.253060] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.253229] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.253393] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.253553] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.253714] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.253873] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.254054] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.service_type = shared-file-system {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.254259] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.share_apply_policy_timeout = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.254426] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.254586] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.254744] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.254901] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.255097] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.255268] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] manila.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.255438] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] mks.enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.255789] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.255982] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] image_cache.manager_interval = 2400 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.256171] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] image_cache.precache_concurrency = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.256352] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] image_cache.remove_unused_base_images = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.256546] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.256731] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.256915] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] image_cache.subdirectory_name = _base {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.257112] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.api_max_retries = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.257284] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.api_retry_interval = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.257452] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.257619] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.auth_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.257780] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.257940] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.258119] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.258286] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.conductor_group = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.258450] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.258611] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.258768] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.258932] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.259103] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.259268] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.259432] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.259598] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.peer_list = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.259756] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.259917] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.260095] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.serial_console_state_timeout = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.260259] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.260435] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.service_type = baremetal {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.260595] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.shard = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.260762] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.260921] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.261093] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.261257] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.261442] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.261608] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ironic.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.261793] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.261970] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] key_manager.fixed_key = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.262170] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.262340] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.barbican_api_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.262502] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.barbican_endpoint = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.262698] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.barbican_endpoint_type = public {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.262884] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.barbican_region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.263061] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.263227] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.263396] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.263558] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.263718] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.263880] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.number_of_retries = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.264054] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.retry_delay = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.264250] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.send_service_user_token = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.264425] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.264584] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.264745] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.verify_ssl = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.264903] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican.verify_ssl_path = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.265082] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.265253] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.auth_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.265414] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.265572] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.265736] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.265896] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.266064] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.266230] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.266393] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] barbican_service_user.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.266578] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.approle_role_id = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.266748] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.approle_secret_id = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.266924] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.kv_mountpoint = secret {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.267095] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.kv_path = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.267266] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.kv_version = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.267428] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.namespace = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.267587] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.root_token_id = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.267745] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.ssl_ca_crt_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.267914] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.timeout = 60.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.268090] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.use_ssl = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.268267] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.268439] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.268598] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.268764] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.268925] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.269096] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.269261] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.269429] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.269589] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.269749] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.269909] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.270078] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.270246] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.270411] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.270585] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.service_type = identity {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.270747] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.270906] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.271075] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.271237] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.271421] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.271581] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] keystone.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.271770] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.ceph_mount_options = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.272091] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.ceph_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.272288] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.connection_uri = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.272458] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_mode = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.272629] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_model_extra_flags = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.272799] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_models = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.272972] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_power_governor_high = performance {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.273158] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_power_governor_low = powersave {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.273326] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_power_management = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.273499] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.273671] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.device_detach_attempts = 8 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.273838] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.device_detach_timeout = 20 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.274014] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.disk_cachemodes = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.274210] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.disk_prefix = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.274373] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.enabled_perf_events = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.274541] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.file_backed_memory = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.274708] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.gid_maps = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.274866] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.hw_disk_discard = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.275035] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.hw_machine_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.275211] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_rbd_ceph_conf = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.275382] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.275566] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.275736] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_rbd_glance_store_name = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.275904] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_rbd_pool = rbd {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.276085] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_type = default {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.276251] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.images_volume_group = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.276419] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.inject_key = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.276606] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.inject_partition = -2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.276777] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.inject_password = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.276940] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.iscsi_iface = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.277122] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.iser_use_multipath = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.277290] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_bandwidth = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.277458] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_completion_timeout = 800 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.277622] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_downtime = 500 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.277784] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_downtime_delay = 75 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.277946] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_downtime_steps = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.278119] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_inbound_addr = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.278286] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_permit_auto_converge = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.278450] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_permit_post_copy = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.278607] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_scheme = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.278778] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_timeout_action = abort {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.278941] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_tunnelled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.279112] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_uri = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.279277] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.live_migration_with_native_tls = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.279437] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.max_queues = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.279599] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.mem_stats_period_seconds = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.279821] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.279986] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.nfs_mount_options = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.280294] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.280474] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.num_aoe_discover_tries = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.280640] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.num_iser_scan_tries = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.280801] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.num_memory_encrypted_guests = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.280964] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.num_nvme_discover_tries = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.281141] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.num_pcie_ports = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.281311] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.num_volume_scan_tries = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.281480] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.pmem_namespaces = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.281639] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.quobyte_client_cfg = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.281934] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.282125] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rbd_connect_timeout = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.282294] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.282459] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.282620] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rbd_secret_uuid = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.282777] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rbd_user = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.282940] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.realtime_scheduler_priority = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.283126] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.remote_filesystem_transport = ssh {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.283290] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rescue_image_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.283456] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rescue_kernel_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.283613] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rescue_ramdisk_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.283782] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rng_dev_path = /dev/urandom {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.283944] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.rx_queue_size = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.284135] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.smbfs_mount_options = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.284463] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.284655] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.snapshot_compression = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.284822] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.snapshot_image_format = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.285054] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.285236] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.sparse_logical_volumes = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.285399] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.swtpm_enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.285569] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.swtpm_group = tss {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.285738] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.swtpm_user = tss {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.285908] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.sysinfo_serial = unique {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.286080] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.tb_cache_size = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.286247] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.tx_queue_size = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.286416] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.uid_maps = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.286611] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.use_virtio_for_bridges = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.286799] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.virt_type = kvm {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.286974] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.volume_clear = zero {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.287157] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.volume_clear_size = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.287331] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.volume_use_multipath = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.287489] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_cache_path = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.287661] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.287831] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_mount_group = qemu {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.287997] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_mount_opts = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.288183] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.288478] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.288663] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.vzstorage_mount_user = stack {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.288890] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.289092] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.289279] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.auth_type = password {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.289444] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.289604] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.289768] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.289933] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.290104] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.290278] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.default_floating_pool = public {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.290440] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.290605] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.extension_sync_interval = 600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.290769] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.http_retries = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.290930] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.291101] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.291265] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.291443] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.metadata_proxy_shared_secret = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.291602] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.291771] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.ovs_bridge = br-int {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.291936] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.physnets = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.292123] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.region_name = RegionOne {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.292288] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.292459] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.service_metadata_proxy = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.292619] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.292787] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.service_type = network {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.292951] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.293124] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.293287] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.293449] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.293630] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.293795] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] neutron.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.293969] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] notifications.bdms_in_notifications = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.294171] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] notifications.default_level = INFO {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.294369] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] notifications.include_share_mapping = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.294556] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] notifications.notification_format = unversioned {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.294723] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] notifications.notify_on_state_change = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.294904] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.295097] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] pci.alias = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.295290] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] pci.device_spec = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.295477] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] pci.report_in_placement = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.295656] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.295834] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.auth_type = password {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.296022] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.auth_url = http://10.180.1.21/identity {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.296182] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.296343] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.296528] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.296706] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.296870] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.297043] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.default_domain_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.297211] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.default_domain_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.297376] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.domain_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.297534] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.domain_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.297692] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.297855] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.298018] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.298182] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.298340] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.298510] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.password = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.298669] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.project_domain_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.298834] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.project_domain_name = Default {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.299006] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.project_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.299187] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.project_name = service {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.299359] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.region_name = RegionOne {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.299522] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.299682] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.299851] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.service_type = placement {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300020] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300187] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300353] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300512] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.system_scope = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300669] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300827] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.trust_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.300984] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.user_domain_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.301166] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.user_domain_name = Default {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.301329] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.user_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.301501] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.username = nova {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.301682] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.301843] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] placement.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.302029] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.cores = 20 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.302201] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.count_usage_from_placement = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.302376] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.302543] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.injected_file_content_bytes = 10240 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.302707] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.injected_file_path_length = 255 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.302869] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.injected_files = 5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.303044] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.instances = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.303214] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.key_pairs = 100 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.303407] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.metadata_items = 128 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.303589] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.ram = 51200 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.303754] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.recheck_quota = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.303919] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.server_group_members = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.304097] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.server_groups = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.304339] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.unified_limits_resource_list = ['servers'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.304525] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] quota.unified_limits_resource_strategy = require {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.304705] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.304873] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.305050] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.image_metadata_prefilter = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.305221] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.305388] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.max_attempts = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.305551] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.max_placement_results = 1000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.305723] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.305888] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.query_placement_for_image_type_support = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.306062] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.306241] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] scheduler.workers = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.306416] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.306618] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.306810] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.306985] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.307168] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.307337] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.307502] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.307688] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.307856] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.host_subset_size = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.308032] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.308200] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.image_properties_default_architecture = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.308366] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.308532] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.isolated_hosts = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.308694] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.isolated_images = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.308871] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.max_instances_per_host = 50 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.309043] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.309211] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.309374] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.pci_in_placement = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.309535] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.309694] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.309855] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.310023] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.310191] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.310356] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.310513] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.track_instance_changes = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.310689] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.310859] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metrics.required = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.311031] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metrics.weight_multiplier = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.311201] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metrics.weight_of_unavailable = -10000.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.311369] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] metrics.weight_setting = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.311682] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.311858] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] serial_console.enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.312046] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] serial_console.port_range = 10000:20000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.312225] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.312397] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.312567] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] serial_console.serialproxy_port = 6083 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.312735] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.312907] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.auth_type = password {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.313080] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.313244] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.313408] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.313568] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.313726] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.313896] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.send_service_user_token = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.314070] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.314251] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] service_user.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.314431] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.agent_enabled = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.314599] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.314900] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.315118] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.html5proxy_host = 0.0.0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.315321] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.html5proxy_port = 6082 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.315502] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.image_compression = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.315666] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.jpeg_compression = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.315827] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.playback_compression = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.315990] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.require_secure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.316179] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.server_listen = 127.0.0.1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.316354] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.316543] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.streaming_mode = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.316719] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] spice.zlib_compression = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.316888] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] upgrade_levels.baseapi = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.317076] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] upgrade_levels.compute = auto {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.317243] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] upgrade_levels.conductor = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.317405] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] upgrade_levels.scheduler = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.317575] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.317742] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.auth_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.317903] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.318076] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.318247] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.318413] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.318575] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.318738] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.318897] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vendordata_dynamic_auth.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.319082] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.api_retry_count = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.319248] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.ca_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.319425] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.cache_prefix = devstack-image-cache {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.319594] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.cluster_name = testcl1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.319760] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.connection_pool_size = 10 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.319921] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.console_delay_seconds = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.320105] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.datastore_regex = ^datastore.* {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.320316] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.320509] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.host_password = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.320680] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.host_port = 443 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.320851] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.host_username = administrator@vsphere.local {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.321030] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.insecure = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.321196] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.integration_bridge = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.321364] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.maximum_objects = 100 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.321527] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.pbm_default_policy = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.321684] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.pbm_enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.321839] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.pbm_wsdl_location = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.322014] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.322182] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.serial_port_proxy_uri = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.322344] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.serial_port_service_uri = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.322508] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.task_poll_interval = 0.5 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.322678] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.use_linked_clone = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.322845] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.vnc_keymap = en-us {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.323022] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.vnc_port = 5900 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.323193] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vmware.vnc_port_total = 10000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.323380] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.auth_schemes = ['none'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.323555] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.323845] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.324041] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.324249] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.novncproxy_port = 6080 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.324450] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.server_listen = 127.0.0.1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.324634] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.324799] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.vencrypt_ca_certs = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.324959] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.vencrypt_client_cert = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.325133] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vnc.vencrypt_client_key = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.325330] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.325507] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.disable_deep_image_inspection = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.325674] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.disable_fallback_pcpu_query = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.325836] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.disable_group_policy_check_upcall = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.325997] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.326175] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.disable_rootwrap = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.326341] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.enable_numa_live_migration = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.326522] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.326702] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.326868] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.handle_virt_lifecycle_events = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.327041] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.libvirt_disable_apic = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.327209] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.never_download_image_if_on_rbd = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.327396] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.327572] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.327736] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.327901] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.328074] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.328272] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.328446] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.328609] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.328776] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.328960] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.329197] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.client_socket_timeout = 900 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.329470] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.default_pool_size = 1000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.329736] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.keep_alive = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.329956] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.max_header_line = 16384 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.330174] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.secure_proxy_ssl_header = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.330354] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.ssl_ca_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.330519] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.ssl_cert_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.330680] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.ssl_key_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.330848] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.tcp_keepidle = 600 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.331038] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.331213] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] zvm.ca_file = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.331375] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] zvm.cloud_connector_url = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.331671] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.331850] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] zvm.reachable_timeout = 300 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.332034] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.332310] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.332518] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.connection_string = messaging:// {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.332692] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.enabled = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.332864] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.es_doc_type = notification {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.333046] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.es_scroll_size = 10000 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.333222] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.es_scroll_time = 2m {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.333416] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.filter_error_trace = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.333592] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.hmac_keys = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.333760] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.sentinel_service_name = mymaster {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.333926] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.socket_timeout = 0.1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.334102] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.trace_requests = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.334286] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler.trace_sqlalchemy = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.334472] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler_jaeger.process_tags = {} {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.334631] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler_jaeger.service_name_prefix = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.334792] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] profiler_otlp.service_name_prefix = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.334956] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] remote_debug.host = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.335127] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] remote_debug.port = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.335306] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.335469] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.335631] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.335793] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.335957] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.336134] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.336300] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.336463] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.336625] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.336793] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.hostname = devstack {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.336951] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.337135] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.337306] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.337474] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.337645] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.337808] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.337972] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.338156] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.338322] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.338482] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.338646] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.338808] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.338969] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.339145] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.339307] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.339509] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.339681] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.339845] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.340026] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.340193] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.ssl = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.340364] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.340541] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.340707] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.340873] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.341051] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.ssl_version = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.341217] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.341405] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.341570] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_notifications.retry = -1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.341746] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.341917] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_messaging_notifications.transport_url = **** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.342102] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.auth_section = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.342272] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.auth_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.342435] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.cafile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.342593] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.certfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.342755] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.collect_timing = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.342911] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.connect_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.343079] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.connect_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.343242] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.endpoint_id = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.343414] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.endpoint_interface = publicURL {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.343572] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.endpoint_override = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.343728] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.endpoint_region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.343886] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.endpoint_service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.344053] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.endpoint_service_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.344249] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.insecure = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.344413] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.keyfile = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.344571] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.max_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.344727] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.min_version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.344881] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.region_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.345054] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.retriable_status_codes = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.345224] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.service_name = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.345404] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.service_type = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.345572] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.split_loggers = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.345731] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.status_code_retries = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.345890] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.status_code_retry_delay = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.346059] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.timeout = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.346224] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.valid_interfaces = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.346383] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_limit.version = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.346550] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_reports.file_event_handler = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.346714] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_reports.file_event_handler_interval = 1 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.346873] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] oslo_reports.log_dir = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.347056] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.347223] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_linux_bridge_privileged.group = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.347384] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.347551] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.347716] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.347875] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_linux_bridge_privileged.user = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.348059] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.348228] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_ovs_privileged.group = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.348391] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_ovs_privileged.helper_command = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.348556] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.348722] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.348880] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] vif_plug_ovs_privileged.user = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.349065] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.flat_interface = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.349253] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.349429] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.349602] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.349775] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.349946] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.350126] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.350290] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_linux_bridge.vlan_interface = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.350470] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.350642] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.isolate_vif = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.350814] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.350982] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.351164] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.351337] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.ovsdb_interface = native {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.351523] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] os_vif_ovs.per_port_bridge = False {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.351705] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] privsep_osbrick.capabilities = [21] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.351870] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] privsep_osbrick.group = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.352039] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] privsep_osbrick.helper_command = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.352213] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.352382] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] privsep_osbrick.thread_pool_size = 8 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.352538] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] privsep_osbrick.user = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.352710] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.352870] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] nova_sys_admin.group = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.353039] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] nova_sys_admin.helper_command = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.353210] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.353375] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] nova_sys_admin.thread_pool_size = 8 {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.353530] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] nova_sys_admin.user = None {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2824}} [ 471.353663] env[62506]: DEBUG oslo_service.service [None req-61ae34e7-4a5d-47b3-8538-244fbf3839f2 None None] ******************************************************************************** {{(pid=62506) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2828}} [ 471.354097] env[62506]: INFO nova.service [-] Starting compute node (version 30.1.0) [ 471.857982] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Getting list of instances from cluster (obj){ [ 471.857982] env[62506]: value = "domain-c8" [ 471.857982] env[62506]: _type = "ClusterComputeResource" [ 471.857982] env[62506]: } {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 471.859068] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc03c4de-ffac-4ac0-a485-71195f5f89a5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 471.867805] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Got total of 0 instances {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 471.868418] env[62506]: WARNING nova.virt.vmwareapi.driver [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 471.868883] env[62506]: INFO nova.virt.node [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Generated node identity 432b599d-3cbd-4439-9dff-9721e0cca9b4 [ 471.869136] env[62506]: INFO nova.virt.node [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Wrote node identity 432b599d-3cbd-4439-9dff-9721e0cca9b4 to /opt/stack/data/n-cpu-1/compute_id [ 472.371723] env[62506]: WARNING nova.compute.manager [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Compute nodes ['432b599d-3cbd-4439-9dff-9721e0cca9b4'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 473.377133] env[62506]: INFO nova.compute.manager [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 474.381951] env[62506]: WARNING nova.compute.manager [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 474.382327] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 474.382506] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 474.382653] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 474.382854] env[62506]: DEBUG nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 474.383813] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ff15a01-3f1f-425e-8b00-963731627ea4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 474.391925] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-928545db-f50f-4949-b4a0-027c6d86b11f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 474.406608] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b06f694f-4f56-462e-9542-4fa259a2cf9d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 474.413208] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5024a71d-b1a3-4426-a831-47fa7ce05341 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 474.442713] env[62506]: DEBUG nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181222MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 474.442879] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 474.443092] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 474.945840] env[62506]: WARNING nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] No compute node record for cpu-1:432b599d-3cbd-4439-9dff-9721e0cca9b4: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host 432b599d-3cbd-4439-9dff-9721e0cca9b4 could not be found. [ 475.449164] env[62506]: INFO nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: 432b599d-3cbd-4439-9dff-9721e0cca9b4 [ 476.958790] env[62506]: DEBUG nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 476.959142] env[62506]: DEBUG nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 477.187299] env[62506]: INFO nova.scheduler.client.report [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] [req-f11030f2-66b6-4688-a4ea-da6fd3c708e9] Created resource provider record via placement API for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4 and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 477.205070] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdf2dc3d-13cf-4086-bd03-f15da7d2fe31 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 477.213450] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d23b6333-a590-45f1-910b-d20d742c9bed {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 477.245739] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83fb3f13-97f6-4c73-91a5-a780175b353f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 477.253519] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e80d8648-be91-4698-a138-0959bab8f773 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 477.267587] env[62506]: DEBUG nova.compute.provider_tree [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 477.803090] env[62506]: DEBUG nova.scheduler.client.report [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 477.805868] env[62506]: DEBUG nova.compute.provider_tree [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 0 to 1 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 477.805868] env[62506]: DEBUG nova.compute.provider_tree [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 477.852097] env[62506]: DEBUG nova.compute.provider_tree [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 1 to 2 during operation: update_traits {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 478.357249] env[62506]: DEBUG nova.compute.resource_tracker [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 478.357609] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.914s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 478.357743] env[62506]: DEBUG nova.service [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Creating RPC server for service compute {{(pid=62506) start /opt/stack/nova/nova/service.py:186}} [ 478.372133] env[62506]: DEBUG nova.service [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] Join ServiceGroup membership for this service compute {{(pid=62506) start /opt/stack/nova/nova/service.py:203}} [ 478.372401] env[62506]: DEBUG nova.servicegroup.drivers.db [None req-3b526ede-6143-41d0-9254-ea71bfe4acca None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=62506) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 506.378999] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_power_states {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 506.882198] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Getting list of instances from cluster (obj){ [ 506.882198] env[62506]: value = "domain-c8" [ 506.882198] env[62506]: _type = "ClusterComputeResource" [ 506.882198] env[62506]: } {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 506.883467] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ff4f8f2-3324-47d4-9a9b-93bd209fc541 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 506.892130] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Got total of 0 instances {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 506.892359] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 506.892670] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Getting list of instances from cluster (obj){ [ 506.892670] env[62506]: value = "domain-c8" [ 506.892670] env[62506]: _type = "ClusterComputeResource" [ 506.892670] env[62506]: } {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 506.893542] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b251362-e1ec-4d66-bbfa-890363944d4a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 506.900593] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Got total of 0 instances {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 526.620165] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 526.620603] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 526.620650] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 526.620770] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 527.124141] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 527.124383] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.124622] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.124831] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.125050] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.125245] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.125426] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.125597] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 527.125741] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 527.628631] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.629046] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.629099] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.629253] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 527.630185] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cb483a7-ad60-43e7-9571-50f93e220238 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 527.638417] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46a62f33-d104-46f0-a4f1-0a2a68933397 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 527.652708] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82dbae8e-c42a-41cc-9fc8-a7abe253d3ff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 527.658777] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a04f6fb-7356-40ad-b1e8-2a2e9f9ee073 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 527.686367] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181241MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 527.686500] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.686681] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.707023] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 528.707023] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 528.722101] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ceb7b88d-22ff-4b7c-a768-366b3177f572 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 528.728210] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31167cd6-981c-4c93-bbf0-90614d5c1c36 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 528.758253] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ad88b4b-c3e0-40a3-a075-a0ab10768b70 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 528.769994] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-300f1c44-8d5e-4012-98c1-314721936dfb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 528.780213] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 529.282188] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 529.787032] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 529.787465] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.101s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.773260] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 589.773583] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.279098] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.279264] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 590.279385] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 590.782724] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 590.783136] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.783358] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.783533] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.783712] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.783921] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.784105] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 590.784239] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 590.784382] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.288351] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.288583] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.288755] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 591.288920] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 591.289890] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3b3fff2-78b8-4deb-96b0-115f7645664b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.298540] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e683b904-b460-48c9-bdd1-caa6e6c60e58 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.312465] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32915b6d-1cb3-468d-b730-8c00db048ddb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.318691] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e3db5ac-052a-459a-9e99-1e5a9f04dee9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.346693] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181214MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 591.346846] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.347020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.366672] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 592.366672] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 592.377740] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54fdd9ce-ea73-4a42-9679-5ac2bc8abd74 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.385733] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a58fef6-86bb-4aa1-865e-d7bb3a2ee952 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.415343] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2f24cfe-0b48-4d67-9ca9-b43b9d9bee91 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.422782] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a018406a-e7e2-4f2a-b5a1-5db2e5ba097f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.435889] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 592.939014] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 592.940293] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 592.940473] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.593s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.941613] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.942036] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.942036] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 652.942120] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 653.444750] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 653.444990] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.445155] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.445307] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.445453] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.445593] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.445737] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.445866] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 653.446014] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 653.948927] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 653.949366] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 653.949366] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 653.949482] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 653.950365] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bb36363-65ab-40b1-8918-20442f4a4e3d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.959435] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6cfd90c-4ead-4b94-b6cc-1210e5bf3e2e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.973806] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb4b74c9-d1d9-4fc3-99af-3748b19cff1e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.980139] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-919829e1-3e26-41e3-b64c-bfa727de70ec {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.007972] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181232MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 654.008137] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.008324] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 655.027085] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 655.027374] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 655.040557] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79ba5df8-4648-4a19-8e20-8bd1ed339ace {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.048072] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00d3b374-3339-4405-9d3e-55e0887ade5f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.079510] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29c93bb9-1a65-495a-a073-77f1648c8925 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.086377] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d31dbd44-29ba-46c0-acf4-8f70db74a178 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.099502] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 655.602181] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 655.603477] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 655.603658] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.595s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.268327] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.270618] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.773512] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.773772] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 713.773836] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 714.276877] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 714.277281] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.277322] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.277450] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.277578] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.277716] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.277861] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.277993] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 714.278148] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 714.781751] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 714.782026] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 714.782171] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 714.782328] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 714.783281] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b536e87-19f2-4c3d-b9d0-fe7a8d2b26fb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.791478] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07f76eb7-0d63-4459-8f7e-79e06d1ceb7b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.805531] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59504452-f5c7-48a8-a99f-427fa13f64f8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.811518] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0dcf536f-2841-4ecd-94eb-3ed2eacfdcf3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.839172] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181225MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 714.839331] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 714.839514] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 715.859235] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 715.859506] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 715.873983] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d359bd3b-de46-41f8-ab7c-90f423cde348 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.881631] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93fcbabd-d715-4424-bf62-9aa8e83ddc49 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.911842] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dbe072b-1cb7-43dc-8d60-a1d5676d3aa3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.918787] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4a11633-c8e0-4598-bba7-22bd0dbce424 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.931959] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 716.435424] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 716.436711] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 716.436934] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.597s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 766.613701] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 766.614100] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11745}} [ 767.118104] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] There are 0 instances to clean {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11754}} [ 767.118370] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 767.118527] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances with incomplete migration {{(pid=62506) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11783}} [ 767.621055] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 769.121054] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 769.610315] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 770.611481] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 770.611811] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 770.611864] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 770.611987] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 771.115703] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 771.116031] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 771.116236] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 771.116391] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 771.117303] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b03e3717-2542-4799-abe5-cf95ca54bf13 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.125483] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9fc68de4-2855-4be2-9abc-0f523d583f17 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.140595] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7df463ce-207f-4dd3-8817-1a7009bfba39 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.147070] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f00eb60-552e-4cee-9538-2051b35100f4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.175006] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181234MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 771.175174] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 771.175370] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.193253] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 772.193514] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 772.205653] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f50cb73-ca84-42a5-91b7-79eab7d14599 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.213291] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af5c03f7-3a3c-46fa-9281-7eb770301886 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.242686] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7ed7338-a700-4757-889a-038fdf954014 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.249611] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf22e350-bf3b-409b-a309-edbc95549d01 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.262095] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 772.765141] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 772.766427] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 772.766606] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.591s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 774.760636] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 774.761074] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 774.761074] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 774.761145] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 775.264236] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 775.264481] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 775.264602] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 827.611196] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 829.611373] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 830.611162] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 831.114587] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.114911] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.115032] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.115185] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 831.116131] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-302e3d31-56c8-4f2b-8708-3fca321c74a0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.124454] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdf01dd6-be52-466c-821d-7732e5ad7b83 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.138279] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3325bbd6-1967-4492-9e0c-22be90fd66f4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.144334] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c6ee095-db12-4fb1-925b-65bde0f1ccfe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.172861] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181232MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 831.172988] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.173189] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 832.207178] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 832.207178] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 832.224092] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 832.236720] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 832.236886] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 832.249370] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 832.266598] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 832.278052] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbb5f911-9c95-4c8a-a3dc-04069adb6c08 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.285311] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e35c36eb-d200-4cef-9cc3-ecfde4d64e8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.314314] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7359dfa5-6b78-4696-be20-fa07a3efe4d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.321697] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05885985-d71e-4d48-80c1-19c0caea6f5f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.334632] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 832.838122] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 832.839418] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 832.839604] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.666s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 834.834550] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 834.834920] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.338825] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.339034] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 835.339110] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 835.842234] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 835.842618] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.842618] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.842751] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.842902] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.843042] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 889.611417] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 890.611193] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 890.611360] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 891.114753] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 891.115075] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 891.115186] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 891.115339] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 891.116271] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8033c136-6d13-44c5-ad1d-fda0bfc3044a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.124472] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b38b205-b863-414e-9840-dd502b107e69 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.138347] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44159fdc-acba-4040-8458-96df8217c9f9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.144484] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ab12fa9-aef2-4f89-891f-59da5fc8942e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.174060] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181229MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 891.174209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 891.174389] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 892.192967] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 892.193219] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 892.206256] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e67d1026-a4af-4369-a1e4-4a0c69a5b367 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.213875] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64609297-bac3-450e-a708-f2967096cfe7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.242337] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6e4ab7c-9fbe-4352-94e4-771d5a45a2de {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.249127] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d963f8a-0a9e-4ee7-8f49-817164d299db {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.261754] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 892.764802] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 892.766063] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 892.766248] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.592s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 894.761117] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 894.761517] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 894.761517] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 894.761638] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 895.264700] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 895.264907] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 895.265095] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 895.265262] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 895.265395] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 896.610607] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.612826] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.610901] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 952.610648] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 953.114013] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.114301] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.114480] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 953.114635] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 953.115561] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44c5396a-b1b9-4363-834f-b8bcd287960c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.123586] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5eb914f-fae4-4268-906d-be4160d11211 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.137478] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8e540c8-ab25-4ba3-b64d-124b9df96fcb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.143605] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e51e2a2-819f-4f1b-a795-ac6dbe3b38e8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.171356] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181236MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 953.171495] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.171695] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.189917] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 954.190171] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 954.204443] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8393cde7-c94b-48b1-8ed9-bb54dac09759 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.211961] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8e82158-5a01-4813-899c-2b1868b8f005 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.240553] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef7b635d-9c84-4d44-8d66-a6521ead98d0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.247627] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb640862-ba07-45e6-9185-25e551597b5b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.260743] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 954.764178] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 954.765485] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 954.765695] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.594s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 955.766443] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 955.766843] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 956.272680] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 956.272680] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 956.272680] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 956.773732] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 956.774188] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 956.774274] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 956.774424] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 956.774580] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 956.774710] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1012.611696] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1012.612101] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1012.612101] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1013.115704] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.115945] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.116125] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.116288] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1013.117497] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb6f0e6f-fb22-41e7-89e6-0adf33943e24 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.125658] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-116887f6-466b-4fc9-8052-ed4eb39ced06 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.139944] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebf0acdc-5b97-4047-809a-41fc23433300 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.146509] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e8ac4be-7426-483b-a884-b29a15773bb6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.174823] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181217MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1013.174959] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.175155] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1014.194748] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1014.194748] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1014.207982] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a79d9829-9b39-4f1f-af00-68f0026cf3e8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.215359] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a847c64b-068e-40d4-931e-d2dd40047ad0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.244634] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2521106-9de5-40a7-a53f-0ad2344f4d82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.251298] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b4d8571-a329-430b-b588-5c39fcd8704f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.263596] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1014.766485] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1014.767785] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1014.767968] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.593s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1017.763182] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1017.763566] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1017.763609] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1017.763736] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1018.266608] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1018.266845] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.267025] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.267185] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.267317] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1018.611056] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1069.611533] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1073.114229] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1073.610026] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1073.610308] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1074.113125] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1074.113395] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1074.113563] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1074.113720] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1074.114618] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78d723ba-f2b2-463c-800f-00a53c6145d0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1074.123061] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fcd9b24-c1d0-46a0-b649-5a86df935a3a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1074.136832] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a0af6cc-a3b8-4948-8619-034a41a98888 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1074.143226] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6a782a0-def8-4378-a337-749040bc4b18 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1074.171515] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181232MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1074.171729] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1074.171859] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1075.189852] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1075.190158] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1075.204169] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cdc5316-1b99-4497-bca6-f4231e177880 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1075.212022] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b1fa269-e7eb-499d-8441-cc6035102616 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1075.241953] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d9edb3f-ed4a-4362-ae94-ac019e281dcf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1075.249243] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-943f4f67-4e08-4688-a1ce-a503ec88a9de {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1075.262262] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1075.765347] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1075.766633] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1075.766818] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.595s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1075.767036] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1075.767176] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11745}} [ 1076.269756] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] There are 0 instances to clean {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11754}} [ 1077.265671] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1077.769618] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1077.769618] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1077.770031] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1078.272019] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1078.272374] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1078.272578] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1078.272741] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1078.272875] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1078.613563] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1080.611232] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1080.611682] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1080.611682] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances with incomplete migration {{(pid=62506) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11783}} [ 1106.882463] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_power_states {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1107.385931] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Getting list of instances from cluster (obj){ [ 1107.385931] env[62506]: value = "domain-c8" [ 1107.385931] env[62506]: _type = "ClusterComputeResource" [ 1107.385931] env[62506]: } {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 1107.387037] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a20facba-c4e5-4d1e-90a2-23f85a9e17b7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1107.395816] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Got total of 0 instances {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 1133.127473] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1133.611192] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1135.610642] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1136.114484] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1136.114750] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1136.114910] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1136.115073] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1136.115942] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac364277-3161-42b5-9ad3-1fc4f8f591ce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1136.124237] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80d62bc6-2ab6-4759-bd55-7f81521181d1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1136.138048] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4067a6b4-695f-4f6e-afe0-2c3d6ec731ea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1136.144326] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-359464ce-5898-408e-91b9-9673fe2cd9aa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1136.172692] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181224MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1136.172882] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1136.173062] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1137.292607] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1137.292879] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1137.307962] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1137.319309] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1137.319492] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1137.328100] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1137.343454] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1137.354139] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fafc6b27-2f1a-41e6-99cc-6872b2b9073b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1137.362540] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee7076bb-46a0-40e6-b052-525adb3c452c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1137.391219] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ff6066c-8456-4b46-a660-e861d8d80cee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1137.398245] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-344e499c-a8ce-4397-828b-f73a169a309b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1137.410667] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1137.914069] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1137.915332] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1137.915543] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.742s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1139.916305] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1139.916666] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1139.916709] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1139.916839] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1140.420038] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1140.420295] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1140.420450] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1140.420648] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1140.420794] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1140.611169] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1193.610327] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1194.612145] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1196.611780] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1197.115592] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1197.115852] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1197.115987] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1197.116157] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1197.117068] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64a11574-7873-4a0c-8925-289793a260fb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1197.124983] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d48ffe0-136d-4f0a-b6f0-171243ea22c7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1197.139172] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc505403-97e0-4b90-ae0a-0a00f48f5ce4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1197.145426] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abbf6f0f-3354-4bde-b081-6fa186a0cfd4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1197.173403] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181224MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1197.173581] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1197.173759] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1198.192744] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1198.193038] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1198.205999] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6e574f6-1f77-4551-a67f-f1a3614219e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1198.213438] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11a52a9a-03d6-4550-8736-5cbe50e6e474 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1198.243231] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5dff5eaa-af8a-478a-8dce-95cf54ae763a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1198.250294] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac813d22-7288-4d48-9db0-1809f04e6dac {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1198.262893] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1198.766027] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1198.767286] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1198.767463] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.594s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1200.766973] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1200.767345] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.274410] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.274627] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1201.274774] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1201.778247] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1201.778599] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.778599] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.778812] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.779049] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.779225] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1253.610646] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1254.611449] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1257.611350] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1258.115459] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1258.115857] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1258.116159] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1258.116428] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1258.118123] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99796cfb-cc79-4e32-a115-04edab209543 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1258.127852] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b4fd5d5-8d47-4d6f-a7a5-fa8752c1e4d5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1258.142012] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e10a9263-b6c5-4ebe-b99a-73a01ca0a96a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1258.148647] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0aab72d-5c45-46f9-bc4c-e552f3455486 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1258.177588] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181225MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1258.177745] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1258.177922] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1259.196436] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1259.196699] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1259.212036] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db31916a-fe22-41ca-a6b4-5021c7f10c06 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1259.219479] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74b79e2c-17e6-4d18-89c9-27a74df66a83 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1259.249230] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3796be4-ffd9-4599-bf51-d7d1157658f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1259.256562] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d08209f-bb96-437c-9555-889334e35185 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1259.269219] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1259.772581] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1259.773947] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1259.774144] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.596s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1261.774791] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1261.775271] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1261.775271] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1261.775384] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1262.278529] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1262.278766] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1262.278941] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1262.279118] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1262.279262] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1263.611351] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1315.612937] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1316.611705] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1318.610714] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1319.115136] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1319.115480] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1319.115670] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1319.115825] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1319.116869] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f0b8380-b2af-4467-ad5c-ef64dc06d176 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1319.128020] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfd50f7d-354d-44a8-86cf-e73bf75fb0d6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1319.140765] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f396536-7af4-4db8-9422-57a7e32f9533 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1319.147173] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b7cb9c7-014d-40c4-add1-eab191b6ce66 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1319.176735] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181208MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1319.176911] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1319.177068] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1320.195544] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1320.195794] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1320.208194] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fd5d475-5c14-484b-9abb-4749dc4dc636 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1320.215711] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd29aeb4-bb65-4c4c-8e55-c0c2affe1f0d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1320.244897] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de7cfb7e-ccb8-4dad-a996-7246394559c7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1320.251599] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e81a8855-df66-4b9b-a0a9-7201a224d6d6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1320.264570] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1320.767700] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1320.768978] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1320.769174] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.592s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1322.764280] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1322.764666] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.271107] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.271177] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1323.271280] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1323.774475] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1323.774772] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.774985] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.775039] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.775222] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.775323] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1374.612302] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1377.114103] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1377.611824] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1378.611800] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1379.114555] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1379.114700] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1379.114863] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1379.115032] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1379.115956] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5209d617-1522-4156-a917-00098cda7ce9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1379.124440] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42f18507-11a7-406a-8485-4829ab5a1ecf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1379.139355] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-825d97b6-107e-4b89-8a53-ab69f9f77d24 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1379.145963] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffcfbea0-fa35-49d6-921a-ca6c48796587 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1379.173678] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181232MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1379.173812] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1379.173997] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1380.192563] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1380.192794] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1380.205328] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cee2f0c7-59aa-43cd-8d8a-987d6aa6c255 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.212987] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f33f694-b4e6-4632-aa8f-d24814384662 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.242536] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11f72aff-50a8-4208-9152-604222aee9f3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.249444] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6b5aba6-0000-4d55-8cbc-d9c5ce771e61 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1380.261977] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1380.765288] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1380.766657] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1380.766865] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.593s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1383.611207] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1383.611587] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1383.611587] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1383.611759] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1384.115275] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1384.115536] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1384.115678] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1384.115838] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1384.115969] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1384.116147] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1384.116273] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances with incomplete migration {{(pid=62506) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11783}} [ 1386.114639] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1387.610895] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1387.611302] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11745}} [ 1388.113720] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] There are 0 instances to clean {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11754}} [ 1437.113385] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1437.610633] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1439.611105] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1440.114916] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1440.115180] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1440.115351] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1440.115511] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1440.116496] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-907e3687-92bd-4fe7-8c96-4623d60d789a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1440.124550] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b251dc6-84d1-4f0b-862a-2b11ca84ecb0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1440.138634] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acc681cc-7dbe-47dc-a468-5cc01cc40c82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1440.145126] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79ebe1f0-a989-47c1-bfe4-35fd32b78e13 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1440.174451] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181218MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1440.174634] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1440.174902] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1441.208821] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1441.209096] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1441.224446] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1441.235449] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1441.235618] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1441.244667] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1441.258799] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1441.270229] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d747fa8-45c5-4228-9175-d5103a122c78 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1441.278435] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb483b96-831c-41bb-bdb1-89f90d47b581 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1441.306946] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6383602-3fc0-42dc-8da8-8c0cb09ae52b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1441.313746] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-217aa74e-3f3a-4db0-a300-ddccc49b8582 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1441.327456] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1441.831115] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1441.832362] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1441.832543] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.658s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1442.828218] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1442.828619] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1443.332937] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1443.333119] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1443.610437] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1443.610621] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1443.610738] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1444.113256] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1444.113507] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1444.611262] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1446.611022] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1497.611448] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1498.610896] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1501.611155] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1502.114420] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1502.114672] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1502.114844] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1502.114996] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1502.115960] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a721cdfe-a8c3-4b9d-b076-f9ae08075f80 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1502.124486] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cd1eeda-3b3a-4e9d-af1c-6ea7f1ef615c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1502.138607] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75bb9833-d167-46f7-af21-d5b7146ca194 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1502.145126] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bf1cbaa-5bd1-41c8-8dbd-e916f6f0e71a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1502.174600] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181213MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1502.174720] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1502.174921] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1503.197372] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1503.197372] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1503.215625] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebcfc28c-c6c6-48e3-913e-04a6133e84b5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1503.224087] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51af3b3f-cfea-403f-9aea-2e77f725fd6f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1503.252992] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e85ba17-aaff-43fe-ab49-19533c037084 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1503.260299] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdb69cdc-ff9d-415b-99d4-abbc8db6abb3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1503.273896] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1503.778015] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1503.779585] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1503.779585] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.605s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1504.774592] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1504.775050] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1504.775050] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1504.775179] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1505.611203] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1505.611413] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1505.611413] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1506.117107] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1506.610855] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1507.610583] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1557.610100] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1558.610630] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1561.610776] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1562.114373] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1562.114625] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1562.114780] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1562.114933] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1562.115836] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0c79e80-46ea-4775-9185-08a83ec7f846 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1562.123904] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dc53684-607c-4695-9f2c-f21a408964c1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1562.137945] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-769cb57d-9545-47ee-aed0-db36858ac9bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1562.144380] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55296028-75b1-4677-afbc-d205c4c94f41 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1562.173643] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181218MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1562.173813] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1562.173968] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1563.192116] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1563.192375] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1563.204712] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdad3015-9203-4c5c-8b5f-39494472aa1d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1563.212208] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f34a551-1e9f-4e70-b407-be1593138193 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1563.241008] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-497b6ff6-36f0-4abc-b910-0ed83651690c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1563.247898] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8270e716-d0d7-4adc-8858-c8a11f4e030d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1563.260267] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1563.763309] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1563.764628] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1563.764808] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.591s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1564.759922] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1564.760320] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1565.265637] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1565.265907] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1565.266016] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1566.612471] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.610892] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.611186] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1567.611229] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1568.114657] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1568.610166] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1619.611736] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1619.612150] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1622.610532] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1623.113763] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1623.113998] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1623.114186] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1623.114341] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1623.115266] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acfc4efa-4bba-42b3-9c6b-3e82fd852f76 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1623.123391] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e38ee1d-cc79-4414-b0f1-928be860b2df {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1623.137142] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e27a1588-4996-4b77-960c-177ea25a899d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1623.143214] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f3940e1-d999-488b-acb1-c21b8c2d6316 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1623.171036] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181209MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1623.171192] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1623.171381] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1624.189893] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1624.190195] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1624.205021] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cc65ece-786b-43cc-996d-7e73374024bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1624.211538] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16661d2a-d35d-46ab-bfd2-30c0a4c5b3e6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1624.239912] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64a91979-684f-4bdd-b139-032889065587 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1624.246722] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bccb80a-81a8-46b4-a6e2-1c9433f8b1ac {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1624.259193] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1624.762679] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1624.763967] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1624.764168] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.593s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1625.764862] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1625.765175] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1625.765276] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1626.610602] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1626.610921] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1627.611518] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1627.611877] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1627.611877] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1628.114575] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1629.610681] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1678.610881] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1680.114284] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1680.610540] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1684.605886] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1685.110205] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1685.613080] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1685.613448] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1685.613550] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1685.613641] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1685.614530] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e27dbac-199b-4dd7-a40a-55d50c5cb74f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.622506] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1eb1d3e6-7ecb-4a56-98bf-4e37e111f2b5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.637032] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a86a6fd-c00d-4b3d-b7c1-066b6dee547f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.643290] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3a54848-4cdf-441d-9f03-042a72ad53bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.671248] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181218MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1685.671389] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1685.671570] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1686.690353] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1686.690604] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1686.705100] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e93b1fc-3c61-46f3-ac72-0e89c4ee7c4d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1686.715389] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3ce2ad2-36f4-477b-81bd-caaf9413060d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1686.744545] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b550e95-eb04-4bdf-b873-75b95dc17596 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1686.751641] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c679518a-eed4-4ff5-8dc3-15b7b4d41d57 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1686.764120] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1687.267429] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1687.268731] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1687.268953] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.597s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1687.769671] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1687.770064] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1687.770064] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1687.770221] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1688.611256] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1688.611488] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1688.611633] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1689.114590] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1689.114842] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1689.610813] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1689.612294] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances with incomplete migration {{(pid=62506) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11783}} [ 1692.113405] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1701.614180] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1701.614594] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11745}} [ 1702.118127] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] There are 0 instances to clean {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11754}} [ 1719.884727] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_power_states {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1720.389109] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Getting list of instances from cluster (obj){ [ 1720.389109] env[62506]: value = "domain-c8" [ 1720.389109] env[62506]: _type = "ClusterComputeResource" [ 1720.389109] env[62506]: } {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 1720.390181] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c57bc42-228e-4a63-aca1-1b811c4f2f00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1720.398903] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Got total of 0 instances {{(pid=62506) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 1740.126424] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1742.612782] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1744.609956] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1745.113543] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1745.113786] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1745.113953] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1745.114117] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1745.114973] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6e438f5-7cc3-46de-a2af-aab00ae91df6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1745.122669] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88683371-0931-4563-ac0d-4d697bfb7cf7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1745.136293] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2f3e752-302b-49b5-a11c-81bbc415bbe3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1745.143248] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae899a9a-c00f-48ab-9d0b-36c15c13dff1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1745.172624] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181219MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1745.172767] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1745.172957] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1746.290426] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1746.290724] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1746.305254] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1746.315917] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1746.316101] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1746.324826] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1746.338700] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1746.349065] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7e67c57-d72a-4094-86d4-925bbeaf6b99 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1746.356484] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f5bd5d3-fe84-45ed-91e5-93f075e7b24b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1746.385012] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0818433-85f5-4c80-a573-b1bc8e431eec {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1746.391805] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6672325-ebb7-430e-9319-4315a98aa3dc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1746.405022] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1746.908668] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1746.910077] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1746.910211] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.737s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1748.911629] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1748.912070] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1748.912070] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1748.912188] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1749.611619] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1749.611831] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1749.611954] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1750.114859] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1750.610639] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1753.610031] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1781.304608] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "3aca933d-f1e7-4dbe-a487-f8954da24981" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1781.305253] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "3aca933d-f1e7-4dbe-a487-f8954da24981" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1781.808881] env[62506]: DEBUG nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1782.352272] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1782.352898] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.002s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1782.354447] env[62506]: INFO nova.compute.claims [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1783.434047] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b501b77-23b6-4b6d-a5cf-c4eef1938efc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1783.444577] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5de9dec1-a349-4423-a56f-91655a34011d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1783.501732] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-096cfdad-695e-459a-923f-02728568e82e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1783.509640] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69801c3e-55ee-4e4f-b521-8c48cb4378b1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1783.528918] env[62506]: DEBUG nova.compute.provider_tree [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1783.835731] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "fb36ae87-2f9c-4973-8c0c-34c4446365cc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1783.835987] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "fb36ae87-2f9c-4973-8c0c-34c4446365cc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1784.038101] env[62506]: DEBUG nova.scheduler.client.report [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1784.339567] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1784.543588] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.191s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1784.544350] env[62506]: DEBUG nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1784.880510] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1784.881139] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1784.882315] env[62506]: INFO nova.compute.claims [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1785.051776] env[62506]: DEBUG nova.compute.utils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1785.054021] env[62506]: DEBUG nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 1785.555383] env[62506]: DEBUG nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1785.854950] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "05b2a777-cf0c-4fdf-a024-6949bd016f6f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1785.855211] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "05b2a777-cf0c-4fdf-a024-6949bd016f6f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1785.996595] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0a33a05-7633-4404-a1c8-ce6950d7249b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.007995] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7ab07ba-10fa-450f-9fba-e6f96b7d432a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.042574] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e5594ad-32cd-44dc-8988-1a7152ee1925 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.052819] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-030f430f-5905-4bde-ac7e-b1fec622e8a3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.076437] env[62506]: DEBUG nova.compute.provider_tree [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1786.362244] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1786.512110] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquiring lock "edc70971-dcfe-4575-9af8-d31e77d1bcf1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1786.512418] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Lock "edc70971-dcfe-4575-9af8-d31e77d1bcf1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1786.583777] env[62506]: DEBUG nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1786.590894] env[62506]: DEBUG nova.scheduler.client.report [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1786.900573] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1787.019743] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1787.098840] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.218s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1787.099402] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1787.103265] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.203s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1787.105688] env[62506]: INFO nova.compute.claims [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1787.411225] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1787.411504] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1787.411661] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1787.411839] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1787.411985] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1787.412152] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1787.412369] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1787.412527] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1787.412692] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1787.412850] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1787.413048] env[62506]: DEBUG nova.virt.hardware [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1787.414026] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f124cd7-0718-4a82-be8d-01892a65e40b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.422750] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4293f72-ac74-49fc-9760-f4c4d40456df {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.440610] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff61e6c6-947d-4fd3-ae6d-8e19c2482511 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.459187] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1787.468570] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1787.468980] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-56c01fbb-944b-4f82-9979-0edd7c1c4c73 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.481575] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Created folder: OpenStack in parent group-v4. [ 1787.481575] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating folder: Project (da748de5fa0345b7bf0f1a88cc033cd7). Parent ref: group-v446953. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1787.481894] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-52f34ff7-3469-4c66-8873-a08815c27eb3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.490631] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Created folder: Project (da748de5fa0345b7bf0f1a88cc033cd7) in parent group-v446953. [ 1787.490813] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating folder: Instances. Parent ref: group-v446954. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1787.491151] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d9758c4e-b964-462c-a0e3-74a60c2afcdb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.502510] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Created folder: Instances in parent group-v446954. [ 1787.502510] env[62506]: DEBUG oslo.service.loopingcall [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1787.502510] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 1787.502720] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-dc363ba8-7797-49c1-896d-480f7df46fe5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1787.518643] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1787.518643] env[62506]: value = "task-2190767" [ 1787.518643] env[62506]: _type = "Task" [ 1787.518643] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1787.530788] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190767, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1787.550297] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1787.611668] env[62506]: DEBUG nova.compute.utils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1787.616540] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1787.617385] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1787.699432] env[62506]: DEBUG nova.policy [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd3d4aed209734b54a5860f96368c620d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5120d07a69a240c0b71da679d98063b1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1787.859405] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquiring lock "ca66e019-db83-4510-a049-8e8147537fa5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1787.859835] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Lock "ca66e019-db83-4510-a049-8e8147537fa5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1788.031309] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190767, 'name': CreateVM_Task, 'duration_secs': 0.358577} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1788.031571] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 1788.032601] env[62506]: DEBUG oslo_vmware.service [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5344ca5f-cbe1-4ce5-a2d8-0b6ca05fc9af {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.042868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1788.043145] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1788.043893] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1788.044130] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e4fd281e-3ab0-4400-8c81-e28734a2bb5f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.049667] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1788.049667] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5201e79f-a917-043b-e05d-06195dec578d" [ 1788.049667] env[62506]: _type = "Task" [ 1788.049667] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1788.059723] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5201e79f-a917-043b-e05d-06195dec578d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1788.117685] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1788.231129] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91a915a8-a9aa-40d9-b043-0bf07bc0f038 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.242115] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d89afe15-3cae-45ab-ba83-f3e533135374 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.283138] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d879187-c2b0-47b3-aea2-b47c2702260c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.291891] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67d90e55-5dde-48ca-b328-3eb825429cf7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.311180] env[62506]: DEBUG nova.compute.provider_tree [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1788.364402] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1788.508220] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Successfully created port: f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1788.564063] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1788.564432] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1788.564780] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1788.564780] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1788.565386] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1788.565483] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-7aab4b0f-5593-4624-89d2-f8fe212a19f5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.584461] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1788.584702] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 1788.586445] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1426a993-615a-4d61-b680-2640c060fdf4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.601247] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-cad1f0f9-eb60-41cd-b41b-33317550c570 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1788.606399] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1788.606399] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52db14a4-834a-75c1-83bc-14b95b1c1f4f" [ 1788.606399] env[62506]: _type = "Task" [ 1788.606399] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1788.617224] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52db14a4-834a-75c1-83bc-14b95b1c1f4f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1788.817626] env[62506]: DEBUG nova.scheduler.client.report [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1788.903131] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1789.122121] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Preparing fetch location {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1789.122273] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating directory with path [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1789.125016] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8cfff0d9-2d1b-4ba5-a3dd-af30a94b6fa7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.128525] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "4bac9b51-a7cb-4499-be0d-e404039df5f0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1789.128719] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "4bac9b51-a7cb-4499-be0d-e404039df5f0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1789.131461] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1789.156983] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Created directory with path [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1789.156983] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Fetch image to [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1789.157477] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Downloading image file data 580d9529-f92b-46b3-a9d4-08013a8922b3 to [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk on the data store datastore2 {{(pid=62506) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1789.159617] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5dc7a37a-de3b-47d7-9ce7-41521c789726 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.163908] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1789.164144] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1789.164302] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1789.164484] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1789.164627] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1789.164797] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1789.165023] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1789.165192] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1789.165359] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1789.165522] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1789.165695] env[62506]: DEBUG nova.virt.hardware [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1789.166718] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d0d39aa-8fa9-4db2-b057-4f3d8c6f6c86 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.175559] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed8f2ec0-e6a1-42fe-9535-8a000d78ec09 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.190505] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7c142cc-f82a-4963-ab61-da601166ee54 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.196963] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b4497fd-e792-4f71-b871-200787e2c8ee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.240628] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd76a5eb-cde8-4ec9-94a8-1a5831aa3ea5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.248476] env[62506]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-d1579320-b723-413b-b1d6-4b2f44ed90d2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1789.327470] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.223s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1789.327622] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1789.330917] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.781s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1789.332253] env[62506]: INFO nova.compute.claims [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1789.337254] env[62506]: DEBUG nova.virt.vmwareapi.images [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Downloading image file data 580d9529-f92b-46b3-a9d4-08013a8922b3 to the data store datastore2 {{(pid=62506) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1789.410080] env[62506]: DEBUG oslo_vmware.rw_handles [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=62506) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1789.636989] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1789.838069] env[62506]: DEBUG nova.compute.utils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1789.840803] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1789.841447] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1789.919287] env[62506]: DEBUG nova.policy [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8615160e68af4fcf9e9cd0d9f97a0b3e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0e4a3d4707b74087b11ee299cd02976c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1790.172239] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1790.218341] env[62506]: DEBUG oslo_vmware.rw_handles [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Completed reading data from the image iterator. {{(pid=62506) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1790.218470] env[62506]: DEBUG oslo_vmware.rw_handles [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=62506) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1790.266726] env[62506]: DEBUG nova.virt.vmwareapi.images [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Downloaded image file data 580d9529-f92b-46b3-a9d4-08013a8922b3 to vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk on the data store datastore2 {{(pid=62506) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1790.271325] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Caching image {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1790.271325] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Copying Virtual Disk [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk to [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 1790.271325] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d2767dd8-4013-4800-8b17-eb3531e3fa99 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1790.280140] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1790.280140] env[62506]: value = "task-2190768" [ 1790.280140] env[62506]: _type = "Task" [ 1790.280140] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1790.292842] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190768, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1790.346679] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1790.462164] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9f5cd89-e75b-4c2b-a6ca-8143e75f4418 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1790.471418] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59411681-11a0-4a85-bcbf-4033b2725de2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1790.507190] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94d7533f-bb4a-418b-a360-4fe55afaca42 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1790.511119] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Successfully created port: 1768d8b1-22ff-4739-90ab-8802f299cd38 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1790.518712] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae7e3312-3b2d-499f-9190-9bd599e2b408 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1790.533128] env[62506]: DEBUG nova.compute.provider_tree [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1790.794883] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190768, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1791.037329] env[62506]: DEBUG nova.scheduler.client.report [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1791.293287] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190768, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.764825} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1791.293581] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Copied Virtual Disk [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk to [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 1791.293759] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Deleting the datastore file [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3/tmp-sparse.vmdk {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1791.294107] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-3bf64155-ef7c-44ac-9299-b67f07b9ec16 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.301327] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1791.301327] env[62506]: value = "task-2190769" [ 1791.301327] env[62506]: _type = "Task" [ 1791.301327] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1791.310324] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190769, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1791.363208] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1791.400865] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1791.401152] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1791.401309] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1791.401764] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1791.401900] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1791.402048] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1791.402266] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1791.402537] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1791.402724] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1791.402866] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1791.403045] env[62506]: DEBUG nova.virt.hardware [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1791.403901] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5d533ba-9934-4f05-9a93-128c1b4d0486 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.413598] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc822436-1519-402b-b325-1501d08f808a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.543714] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.213s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1791.544330] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1791.548443] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.645s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1791.550223] env[62506]: INFO nova.compute.claims [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1791.816146] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190769, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.024398} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1791.816146] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1791.816146] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Moving file from [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7/580d9529-f92b-46b3-a9d4-08013a8922b3 to [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3. {{(pid=62506) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 1791.816146] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-b603758e-307e-40f8-8a14-a54c28402dcd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.825570] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1791.825570] env[62506]: value = "task-2190770" [ 1791.825570] env[62506]: _type = "Task" [ 1791.825570] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1791.841432] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.841740] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1791.843221] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190770, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1791.972498] env[62506]: ERROR nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1791.972498] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1791.972498] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1791.972498] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1791.972498] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1791.972498] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1791.972498] env[62506]: ERROR nova.compute.manager raise self.value [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1791.972498] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1791.972498] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1791.972498] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1791.973218] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1791.973218] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1791.973218] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1791.973218] env[62506]: ERROR nova.compute.manager [ 1791.973218] env[62506]: Traceback (most recent call last): [ 1791.973218] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1791.973218] env[62506]: listener.cb(fileno) [ 1791.973218] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1791.973218] env[62506]: result = function(*args, **kwargs) [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1791.973218] env[62506]: return func(*args, **kwargs) [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1791.973218] env[62506]: raise e [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1791.973218] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1791.973218] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1791.973218] env[62506]: with excutils.save_and_reraise_exception(): [ 1791.973218] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1791.973218] env[62506]: self.force_reraise() [ 1791.973218] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1791.973218] env[62506]: raise self.value [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1791.973218] env[62506]: updated_port = self._update_port( [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1791.973218] env[62506]: _ensure_no_port_binding_failure(port) [ 1791.973218] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1791.973218] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1791.974407] env[62506]: nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1791.974407] env[62506]: Removing descriptor: 15 [ 1791.974407] env[62506]: ERROR nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Traceback (most recent call last): [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] yield resources [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self.driver.spawn(context, instance, image_meta, [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1791.974407] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] vm_ref = self.build_virtual_machine(instance, [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] vif_infos = vmwarevif.get_vif_info(self._session, [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] for vif in network_info: [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return self._sync_wrapper(fn, *args, **kwargs) [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self.wait() [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self[:] = self._gt.wait() [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return self._exit_event.wait() [ 1791.975114] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] result = hub.switch() [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return self.greenlet.switch() [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] result = function(*args, **kwargs) [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return func(*args, **kwargs) [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] raise e [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] nwinfo = self.network_api.allocate_for_instance( [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1791.975953] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] created_port_ids = self._update_ports_for_instance( [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] with excutils.save_and_reraise_exception(): [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self.force_reraise() [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] raise self.value [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] updated_port = self._update_port( [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] _ensure_no_port_binding_failure(port) [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1791.976422] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] raise exception.PortBindingFailed(port_id=port['id']) [ 1791.976888] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1791.976888] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] [ 1791.976888] env[62506]: INFO nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Terminating instance [ 1792.055926] env[62506]: DEBUG nova.compute.utils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1792.056535] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1792.056841] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1792.154229] env[62506]: DEBUG nova.policy [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd8be2de556454e7298f24745104e81c7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ba910a5c3bc74b8ba209c9d93fc48476', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1792.344687] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190770, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.0337} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1792.345664] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] File moved {{(pid=62506) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 1792.346307] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Cleaning up location [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 1792.347177] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Deleting the datastore file [datastore2] vmware_temp/4da59eb3-14e7-4e74-9564-8cf278cb33d7 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1792.347623] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1792.351705] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7cad7f6a-fda5-4b03-923b-401f8a74fb31 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.359732] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1792.359732] env[62506]: value = "task-2190771" [ 1792.359732] env[62506]: _type = "Task" [ 1792.359732] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1792.371727] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190771, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1792.479912] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1792.480460] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquired lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1792.480460] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1792.560740] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1792.705383] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10046ca2-e279-421e-a97f-693c8fbee2e2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.715195] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c3a9f66-59e5-407d-acf3-cc8d0d0407f0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.721871] env[62506]: DEBUG nova.compute.manager [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Received event network-changed-f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1792.722089] env[62506]: DEBUG nova.compute.manager [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Refreshing instance network info cache due to event network-changed-f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1792.722304] env[62506]: DEBUG oslo_concurrency.lockutils [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] Acquiring lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1792.759580] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-773ba149-e073-4740-8bea-7059b3660612 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.767875] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efabe2bc-14c3-4bb5-9c89-fdbeff502cfa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.785210] env[62506]: DEBUG nova.compute.provider_tree [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1792.876033] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190771, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.027731} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1792.876033] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1792.876033] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4daf6e0e-9e15-4868-85cb-da398ea9854d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.881033] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1792.881033] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c9688c-f532-73c4-629c-26e1de10d5d1" [ 1792.881033] env[62506]: _type = "Task" [ 1792.881033] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1792.884016] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1792.897436] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52c9688c-f532-73c4-629c-26e1de10d5d1, 'name': SearchDatastore_Task, 'duration_secs': 0.008838} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1792.897436] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1792.897436] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981/3aca933d-f1e7-4dbe-a487-f8954da24981.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 1792.897436] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-3a243ddb-3ec6-4783-b59c-98e1e3cd7d11 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1792.901386] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1792.901386] env[62506]: value = "task-2190772" [ 1792.901386] env[62506]: _type = "Task" [ 1792.901386] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1792.910323] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190772, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1793.041919] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1793.230631] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1793.329840] env[62506]: ERROR nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [req-821e473e-9824-4455-af67-ee5a8995e861] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-821e473e-9824-4455-af67-ee5a8995e861"}]} [ 1793.332044] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Successfully created port: f3c5e5d1-9872-4613-9745-72ee1b87647d {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1793.359116] env[62506]: DEBUG nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1793.383631] env[62506]: DEBUG nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1793.383848] env[62506]: DEBUG nova.compute.provider_tree [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1793.388266] env[62506]: ERROR nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1793.388266] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1793.388266] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1793.388266] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1793.388266] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1793.388266] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1793.388266] env[62506]: ERROR nova.compute.manager raise self.value [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1793.388266] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1793.388266] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1793.388266] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1793.388982] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1793.388982] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1793.388982] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1793.388982] env[62506]: ERROR nova.compute.manager [ 1793.388982] env[62506]: Traceback (most recent call last): [ 1793.388982] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1793.388982] env[62506]: listener.cb(fileno) [ 1793.388982] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1793.388982] env[62506]: result = function(*args, **kwargs) [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1793.388982] env[62506]: return func(*args, **kwargs) [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1793.388982] env[62506]: raise e [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1793.388982] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1793.388982] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1793.388982] env[62506]: with excutils.save_and_reraise_exception(): [ 1793.388982] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1793.388982] env[62506]: self.force_reraise() [ 1793.388982] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1793.388982] env[62506]: raise self.value [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1793.388982] env[62506]: updated_port = self._update_port( [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1793.388982] env[62506]: _ensure_no_port_binding_failure(port) [ 1793.388982] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1793.388982] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1793.389874] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1793.389874] env[62506]: Removing descriptor: 19 [ 1793.389874] env[62506]: ERROR nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Traceback (most recent call last): [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] yield resources [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self.driver.spawn(context, instance, image_meta, [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1793.389874] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] vm_ref = self.build_virtual_machine(instance, [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] for vif in network_info: [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return self._sync_wrapper(fn, *args, **kwargs) [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self.wait() [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self[:] = self._gt.wait() [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return self._exit_event.wait() [ 1793.390345] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] result = hub.switch() [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return self.greenlet.switch() [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] result = function(*args, **kwargs) [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return func(*args, **kwargs) [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] raise e [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] nwinfo = self.network_api.allocate_for_instance( [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1793.390730] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] created_port_ids = self._update_ports_for_instance( [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] with excutils.save_and_reraise_exception(): [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self.force_reraise() [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] raise self.value [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] updated_port = self._update_port( [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] _ensure_no_port_binding_failure(port) [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1793.391229] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] raise exception.PortBindingFailed(port_id=port['id']) [ 1793.391526] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1793.391526] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] [ 1793.391526] env[62506]: INFO nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Terminating instance [ 1793.401279] env[62506]: DEBUG nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1793.413245] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190772, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.488427} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1793.413512] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981/3aca933d-f1e7-4dbe-a487-f8954da24981.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 1793.414175] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 1793.414175] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-84f001f8-0e7c-4251-9534-beda14de7e90 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.421601] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1793.421601] env[62506]: value = "task-2190773" [ 1793.421601] env[62506]: _type = "Task" [ 1793.421601] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1793.421857] env[62506]: DEBUG nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1793.437102] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190773, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1793.567800] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f56b0abd-dcb1-43e9-9bff-7344aaaa0ffd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.572994] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1793.582603] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4815e51a-a963-41d1-aa40-0354d1108511 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.623696] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88bbb69b-c7de-4b24-8b4f-ddb3c927f0b7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.632148] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c0f0e5c-2b33-4c06-afed-10f8412a6c42 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.637720] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1793.637969] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1793.638145] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1793.638327] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1793.638469] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1793.638611] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1793.638815] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1793.638995] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1793.639187] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1793.639353] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1793.639523] env[62506]: DEBUG nova.virt.hardware [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1793.640382] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80300d65-061d-4cfd-937e-83426e790f50 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.652886] env[62506]: DEBUG nova.compute.provider_tree [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1793.658483] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e66a30d3-ca81-4835-80bd-b26e9f135b65 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.737357] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Releasing lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1793.737357] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1793.737537] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1793.737813] env[62506]: DEBUG oslo_concurrency.lockutils [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] Acquired lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1793.737982] env[62506]: DEBUG nova.network.neutron [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Refreshing network info cache for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1793.739102] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fde1eaf7-028a-45af-8643-b3e5d8146ffe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.751783] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3abc394-38a4-4265-b019-5af9eea30ec0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.776045] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fb36ae87-2f9c-4973-8c0c-34c4446365cc could not be found. [ 1793.776291] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1793.776910] env[62506]: INFO nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1793.777200] env[62506]: DEBUG oslo.service.loopingcall [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1793.777526] env[62506]: DEBUG nova.compute.manager [-] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1793.777681] env[62506]: DEBUG nova.network.neutron [-] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1793.825679] env[62506]: DEBUG nova.network.neutron [-] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1793.895048] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1793.895331] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1793.895331] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1793.933220] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190773, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.179926} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1793.933664] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 1793.934574] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ceb007be-9b91-4462-ac3f-f625228d28b7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.957388] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Reconfiguring VM instance instance-00000001 to attach disk [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981/3aca933d-f1e7-4dbe-a487-f8954da24981.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 1793.957731] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-dd2adbb0-1d3a-4626-a0eb-f2965caaed63 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1793.979379] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1793.979379] env[62506]: value = "task-2190774" [ 1793.979379] env[62506]: _type = "Task" [ 1793.979379] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1793.988762] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190774, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1794.211961] env[62506]: DEBUG nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 11 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1794.212732] env[62506]: DEBUG nova.compute.provider_tree [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 11 to 12 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1794.212977] env[62506]: DEBUG nova.compute.provider_tree [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1794.305492] env[62506]: DEBUG nova.network.neutron [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1794.333811] env[62506]: DEBUG nova.network.neutron [-] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1794.461387] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1794.469749] env[62506]: DEBUG nova.network.neutron [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1794.496987] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190774, 'name': ReconfigVM_Task, 'duration_secs': 0.331171} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1794.496987] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Reconfigured VM instance instance-00000001 to attach disk [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981/3aca933d-f1e7-4dbe-a487-f8954da24981.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1794.496987] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-1092f5cc-79af-456c-b4f4-5ad706904a87 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1794.506515] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1794.506515] env[62506]: value = "task-2190775" [ 1794.506515] env[62506]: _type = "Task" [ 1794.506515] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1794.516096] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190775, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1794.709528] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1794.720293] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.170s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1794.720293] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1794.725705] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.555s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1794.728825] env[62506]: INFO nova.compute.claims [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1794.836631] env[62506]: INFO nova.compute.manager [-] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Took 1.06 seconds to deallocate network for instance. [ 1794.839156] env[62506]: DEBUG nova.compute.claims [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1794.839576] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1794.915702] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquiring lock "54e34c6f-3863-4683-b301-a6a9d25f1c4d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1794.916151] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Lock "54e34c6f-3863-4683-b301-a6a9d25f1c4d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1794.972445] env[62506]: DEBUG oslo_concurrency.lockutils [req-e2f399a2-d195-4a92-8c30-394ff386b9e3 req-5dc98e9f-8c3a-46a6-82c0-664202dc0acd service nova] Releasing lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1795.015395] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190775, 'name': Rename_Task} progress is 14%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1795.212283] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1795.215145] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1795.215145] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1795.215145] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ab44da53-9aa3-4d53-b84e-13abf60a65a6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1795.224781] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9480eb95-1c08-48f3-a33e-bb9a72001346 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1795.241357] env[62506]: DEBUG nova.compute.utils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1795.250217] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1795.250217] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1795.267962] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 05b2a777-cf0c-4fdf-a024-6949bd016f6f could not be found. [ 1795.269380] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1795.269986] env[62506]: INFO nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1795.269986] env[62506]: DEBUG oslo.service.loopingcall [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1795.270507] env[62506]: DEBUG nova.compute.manager [-] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1795.270604] env[62506]: DEBUG nova.network.neutron [-] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1795.319037] env[62506]: DEBUG nova.network.neutron [-] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1795.353419] env[62506]: DEBUG nova.policy [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6c4c10a73d424dc5ba491ee28d09917f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7eb1390f003841e1a8b57f6e7f55125f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1795.418859] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1795.516314] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190775, 'name': Rename_Task, 'duration_secs': 0.888497} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1795.516586] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 1795.516844] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-e8bd308f-a1d4-4a4f-b883-3378347e3c41 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1795.524365] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1795.524365] env[62506]: value = "task-2190776" [ 1795.524365] env[62506]: _type = "Task" [ 1795.524365] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1795.532465] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190776, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1795.659025] env[62506]: DEBUG nova.compute.manager [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Received event network-vif-deleted-f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1795.659025] env[62506]: DEBUG nova.compute.manager [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Received event network-changed-1768d8b1-22ff-4739-90ab-8802f299cd38 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1795.659025] env[62506]: DEBUG nova.compute.manager [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Refreshing instance network info cache due to event network-changed-1768d8b1-22ff-4739-90ab-8802f299cd38. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1795.659025] env[62506]: DEBUG oslo_concurrency.lockutils [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] Acquiring lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1795.659025] env[62506]: DEBUG oslo_concurrency.lockutils [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] Acquired lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1795.659622] env[62506]: DEBUG nova.network.neutron [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Refreshing network info cache for port 1768d8b1-22ff-4739-90ab-8802f299cd38 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1795.746681] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1795.822653] env[62506]: DEBUG nova.network.neutron [-] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1795.952669] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1795.976939] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e69b2132-2982-43cf-8eda-119b822e5a25 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1795.991999] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-615eb258-c686-43cb-8886-898c79b1b8fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1796.029979] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e197969-d16c-4fe9-bfca-c59764c0860f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1796.044026] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3cad90b-5695-4664-8fc0-d4476b5b0546 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1796.046140] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190776, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1796.059248] env[62506]: DEBUG nova.compute.provider_tree [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1796.228281] env[62506]: DEBUG nova.network.neutron [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1796.324977] env[62506]: INFO nova.compute.manager [-] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Took 1.05 seconds to deallocate network for instance. [ 1796.331296] env[62506]: DEBUG nova.compute.claims [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1796.331519] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1796.433184] env[62506]: ERROR nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1796.433184] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1796.433184] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1796.433184] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1796.433184] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1796.433184] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1796.433184] env[62506]: ERROR nova.compute.manager raise self.value [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1796.433184] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1796.433184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1796.433184] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1796.433652] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1796.433652] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1796.433652] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1796.433652] env[62506]: ERROR nova.compute.manager [ 1796.434578] env[62506]: Traceback (most recent call last): [ 1796.435142] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1796.435142] env[62506]: listener.cb(fileno) [ 1796.435142] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1796.435142] env[62506]: result = function(*args, **kwargs) [ 1796.435142] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1796.435142] env[62506]: return func(*args, **kwargs) [ 1796.435142] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1796.435142] env[62506]: raise e [ 1796.435142] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1796.435142] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1796.435142] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1796.435142] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1796.435142] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1796.435142] env[62506]: with excutils.save_and_reraise_exception(): [ 1796.435142] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1796.435142] env[62506]: self.force_reraise() [ 1796.435557] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1796.435557] env[62506]: raise self.value [ 1796.435557] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1796.435557] env[62506]: updated_port = self._update_port( [ 1796.435557] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1796.435557] env[62506]: _ensure_no_port_binding_failure(port) [ 1796.435557] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1796.435557] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1796.435557] env[62506]: nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1796.435557] env[62506]: Removing descriptor: 15 [ 1796.436473] env[62506]: ERROR nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Traceback (most recent call last): [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] yield resources [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self.driver.spawn(context, instance, image_meta, [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] vm_ref = self.build_virtual_machine(instance, [ 1796.436473] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] vif_infos = vmwarevif.get_vif_info(self._session, [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] for vif in network_info: [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return self._sync_wrapper(fn, *args, **kwargs) [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self.wait() [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self[:] = self._gt.wait() [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return self._exit_event.wait() [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1796.436836] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] result = hub.switch() [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return self.greenlet.switch() [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] result = function(*args, **kwargs) [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return func(*args, **kwargs) [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] raise e [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] nwinfo = self.network_api.allocate_for_instance( [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] created_port_ids = self._update_ports_for_instance( [ 1796.437207] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] with excutils.save_and_reraise_exception(): [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self.force_reraise() [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] raise self.value [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] updated_port = self._update_port( [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] _ensure_no_port_binding_failure(port) [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] raise exception.PortBindingFailed(port_id=port['id']) [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1796.438349] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] [ 1796.439548] env[62506]: INFO nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Terminating instance [ 1796.451460] env[62506]: DEBUG nova.network.neutron [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1796.539318] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190776, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1796.563584] env[62506]: DEBUG nova.scheduler.client.report [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1796.738028] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Successfully created port: 7f21cb96-831a-42cb-890e-8cefb13ac7a5 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1796.765126] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1796.812263] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1796.814302] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1796.814302] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1796.814302] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1796.814302] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1796.814302] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1796.814466] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1796.814466] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1796.814466] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1796.814466] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1796.814466] env[62506]: DEBUG nova.virt.hardware [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1796.814957] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-978534c2-ade5-4bd5-a88c-30fa8c0f5c2e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1796.826451] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4712670-8526-415e-b371-78544e192fa7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1796.947923] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquiring lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1796.947923] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquired lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1796.947923] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1796.954292] env[62506]: DEBUG oslo_concurrency.lockutils [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] Releasing lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1796.955942] env[62506]: DEBUG nova.compute.manager [req-9378f1a2-f999-4bf5-bb4d-1bb7dada896d req-1b151243-f56e-4a98-ab25-7c1fe0af0068 service nova] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Received event network-vif-deleted-1768d8b1-22ff-4739-90ab-8802f299cd38 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1797.042592] env[62506]: DEBUG oslo_vmware.api [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190776, 'name': PowerOnVM_Task, 'duration_secs': 1.253289} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1797.042592] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 1797.042592] env[62506]: INFO nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Took 10.46 seconds to spawn the instance on the hypervisor. [ 1797.042592] env[62506]: DEBUG nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1797.042592] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e9634d3-3a4e-44dd-bbea-199104caa11f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1797.073095] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.348s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1797.073587] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1797.076165] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.194s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1797.077811] env[62506]: INFO nova.compute.claims [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1797.474365] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1797.571663] env[62506]: INFO nova.compute.manager [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Took 15.26 seconds to build instance. [ 1797.582603] env[62506]: DEBUG nova.compute.utils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1797.589497] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1797.590456] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1797.756432] env[62506]: DEBUG nova.policy [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '51004eee6b49429585b31b69771784a2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bba4ac4e183143928b50b606339514f5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1797.765174] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1798.075901] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ee395e55-50ab-4caa-9820-202078cbb1b3 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "3aca933d-f1e7-4dbe-a487-f8954da24981" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 16.769s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1798.090496] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1798.184143] env[62506]: ERROR nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1798.184143] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1798.184143] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1798.184143] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1798.184143] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1798.184143] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1798.184143] env[62506]: ERROR nova.compute.manager raise self.value [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1798.184143] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1798.184143] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1798.184143] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1798.184673] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1798.184673] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1798.184673] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1798.184673] env[62506]: ERROR nova.compute.manager [ 1798.184673] env[62506]: Traceback (most recent call last): [ 1798.184673] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1798.184673] env[62506]: listener.cb(fileno) [ 1798.184673] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1798.184673] env[62506]: result = function(*args, **kwargs) [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1798.184673] env[62506]: return func(*args, **kwargs) [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1798.184673] env[62506]: raise e [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1798.184673] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1798.184673] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1798.184673] env[62506]: with excutils.save_and_reraise_exception(): [ 1798.184673] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1798.184673] env[62506]: self.force_reraise() [ 1798.184673] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1798.184673] env[62506]: raise self.value [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1798.184673] env[62506]: updated_port = self._update_port( [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1798.184673] env[62506]: _ensure_no_port_binding_failure(port) [ 1798.184673] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1798.184673] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1798.185413] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1798.185413] env[62506]: Removing descriptor: 17 [ 1798.185592] env[62506]: ERROR nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] Traceback (most recent call last): [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] yield resources [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self.driver.spawn(context, instance, image_meta, [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] vm_ref = self.build_virtual_machine(instance, [ 1798.185592] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] vif_infos = vmwarevif.get_vif_info(self._session, [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] for vif in network_info: [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return self._sync_wrapper(fn, *args, **kwargs) [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self.wait() [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self[:] = self._gt.wait() [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return self._exit_event.wait() [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1798.188046] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] result = hub.switch() [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return self.greenlet.switch() [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] result = function(*args, **kwargs) [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return func(*args, **kwargs) [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] raise e [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] nwinfo = self.network_api.allocate_for_instance( [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] created_port_ids = self._update_ports_for_instance( [ 1798.188528] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] with excutils.save_and_reraise_exception(): [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self.force_reraise() [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] raise self.value [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] updated_port = self._update_port( [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] _ensure_no_port_binding_failure(port) [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] raise exception.PortBindingFailed(port_id=port['id']) [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1798.188952] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] [ 1798.192053] env[62506]: INFO nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Terminating instance [ 1798.258720] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ff945e0-3d8f-4cd0-acf5-a36144f6ff69 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1798.268802] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75ef19df-bbb9-46ea-9a17-1c51e392ed25 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1798.272518] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Releasing lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1798.272987] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1798.273577] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1798.274341] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5befc420-3a12-4a30-95f9-fa3c8514b311 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1798.311428] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e231ce96-070a-4b5f-87bb-7a4b2fe297a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1798.316661] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad1b1933-0dae-47c8-b1f2-9d705b77e4c7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1798.333945] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e716d83-80c6-415c-a373-c1888450e40c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1798.345566] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance edc70971-dcfe-4575-9af8-d31e77d1bcf1 could not be found. [ 1798.345566] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1798.345566] env[62506]: INFO nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Took 0.07 seconds to destroy the instance on the hypervisor. [ 1798.345566] env[62506]: DEBUG oslo.service.loopingcall [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1798.345566] env[62506]: DEBUG nova.compute.manager [-] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1798.345566] env[62506]: DEBUG nova.network.neutron [-] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1798.354263] env[62506]: DEBUG nova.compute.provider_tree [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1798.401025] env[62506]: DEBUG nova.network.neutron [-] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1798.693182] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquiring lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1798.693524] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquired lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1798.693602] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1798.857295] env[62506]: DEBUG nova.scheduler.client.report [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1798.904976] env[62506]: DEBUG nova.network.neutron [-] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1799.034279] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Successfully created port: 2c1994e5-967d-42ea-932c-c787807208fe {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1799.107907] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1799.142648] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1799.142796] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1799.142855] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1799.143339] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1799.143414] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1799.143757] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1799.143857] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1799.144020] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1799.144196] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1799.144564] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1799.144813] env[62506]: DEBUG nova.virt.hardware [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1799.145697] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1d8e670-0e8c-448b-a840-435e9709172c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1799.155846] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe499304-d42f-4ac1-b125-fa245b4dd500 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1799.229355] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1799.363080] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.286s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1799.366026] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1799.367623] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 4.528s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1799.412096] env[62506]: INFO nova.compute.manager [-] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Took 1.07 seconds to deallocate network for instance. [ 1799.417040] env[62506]: DEBUG nova.compute.claims [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1799.417040] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1799.475522] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1799.611955] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1799.876499] env[62506]: DEBUG nova.compute.utils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1799.881419] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1799.881725] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1799.984652] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Releasing lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1799.984652] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1799.984652] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1799.984652] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5856da3e-f2ee-497c-b295-c312fe85c807 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1799.997431] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f3a33c5-5b11-40cc-9903-a6705bb612cb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1800.035107] env[62506]: DEBUG nova.policy [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '51004eee6b49429585b31b69771784a2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bba4ac4e183143928b50b606339514f5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1800.043879] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ca66e019-db83-4510-a049-8e8147537fa5 could not be found. [ 1800.044131] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1800.044317] env[62506]: INFO nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1800.044560] env[62506]: DEBUG oslo.service.loopingcall [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1800.044810] env[62506]: DEBUG nova.compute.manager [-] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1800.044899] env[62506]: DEBUG nova.network.neutron [-] [instance: ca66e019-db83-4510-a049-8e8147537fa5] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1800.092223] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a1ef41b-782d-4987-b736-f3e262c7a091 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1800.100167] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8ff68cc-633a-4828-bdee-05144326f9de {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1800.140605] env[62506]: DEBUG nova.network.neutron [-] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1800.143451] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ace19d30-69a9-43a2-b0ed-52e2a6c0ac0f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1800.153147] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f16b1c5-376f-4e63-a64a-70d6ab8864a6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1800.169072] env[62506]: DEBUG nova.compute.provider_tree [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1800.383042] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1800.644566] env[62506]: DEBUG nova.network.neutron [-] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1800.673922] env[62506]: DEBUG nova.scheduler.client.report [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1800.776069] env[62506]: DEBUG nova.compute.manager [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Received event network-changed-f3c5e5d1-9872-4613-9745-72ee1b87647d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1800.776264] env[62506]: DEBUG nova.compute.manager [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Refreshing instance network info cache due to event network-changed-f3c5e5d1-9872-4613-9745-72ee1b87647d. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1800.776472] env[62506]: DEBUG oslo_concurrency.lockutils [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] Acquiring lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1800.776664] env[62506]: DEBUG oslo_concurrency.lockutils [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] Acquired lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1800.776754] env[62506]: DEBUG nova.network.neutron [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Refreshing network info cache for port f3c5e5d1-9872-4613-9745-72ee1b87647d {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1800.829787] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Successfully created port: d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1801.119297] env[62506]: DEBUG nova.compute.manager [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Received event network-changed-7f21cb96-831a-42cb-890e-8cefb13ac7a5 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1801.119400] env[62506]: DEBUG nova.compute.manager [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Refreshing instance network info cache due to event network-changed-7f21cb96-831a-42cb-890e-8cefb13ac7a5. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1801.119596] env[62506]: DEBUG oslo_concurrency.lockutils [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] Acquiring lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1801.119717] env[62506]: DEBUG oslo_concurrency.lockutils [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] Acquired lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1801.119871] env[62506]: DEBUG nova.network.neutron [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Refreshing network info cache for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1801.146607] env[62506]: INFO nova.compute.manager [-] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Took 1.10 seconds to deallocate network for instance. [ 1801.149064] env[62506]: DEBUG nova.compute.claims [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1801.149314] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1801.179979] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.812s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1801.181357] env[62506]: ERROR nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Traceback (most recent call last): [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self.driver.spawn(context, instance, image_meta, [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] vm_ref = self.build_virtual_machine(instance, [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] vif_infos = vmwarevif.get_vif_info(self._session, [ 1801.181357] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] for vif in network_info: [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return self._sync_wrapper(fn, *args, **kwargs) [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self.wait() [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self[:] = self._gt.wait() [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return self._exit_event.wait() [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] result = hub.switch() [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1801.181712] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return self.greenlet.switch() [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] result = function(*args, **kwargs) [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] return func(*args, **kwargs) [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] raise e [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] nwinfo = self.network_api.allocate_for_instance( [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] created_port_ids = self._update_ports_for_instance( [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] with excutils.save_and_reraise_exception(): [ 1801.182089] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] self.force_reraise() [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] raise self.value [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] updated_port = self._update_port( [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] _ensure_no_port_binding_failure(port) [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] raise exception.PortBindingFailed(port_id=port['id']) [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] nova.exception.PortBindingFailed: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. [ 1801.182426] env[62506]: ERROR nova.compute.manager [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] [ 1801.182692] env[62506]: DEBUG nova.compute.utils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1801.182724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 5.230s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1801.185805] env[62506]: INFO nova.compute.claims [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1801.190380] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Build of instance fb36ae87-2f9c-4973-8c0c-34c4446365cc was re-scheduled: Binding failed for port f85d92c2-00b2-4ac9-afaa-9a4d68d93a8b, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1801.190877] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1801.191252] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1801.191463] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquired lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1801.191667] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1801.210675] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquiring lock "c16e39bd-3443-4dbb-8d5a-ab90cf27251f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1801.210992] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Lock "c16e39bd-3443-4dbb-8d5a-ab90cf27251f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1801.335545] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquiring lock "49423ca9-a31f-44d0-b05e-ab78b6b31321" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1801.335853] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Lock "49423ca9-a31f-44d0-b05e-ab78b6b31321" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1801.364084] env[62506]: DEBUG nova.network.neutron [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1801.397974] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1801.427928] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1801.428783] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1801.429074] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1801.429427] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1801.429714] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1801.430019] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1801.430461] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1801.430809] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1801.431320] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1801.431585] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1801.431808] env[62506]: DEBUG nova.virt.hardware [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1801.433341] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5738ffb5-454b-4f61-a25f-fc0ba2dc5bff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1801.447149] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cf8e847-5226-4b5c-9517-ea048c2920fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1801.595323] env[62506]: DEBUG nova.compute.manager [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1801.596626] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c1b8e87-6cff-485f-b61b-7042b3ae2c5b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1801.665610] env[62506]: DEBUG nova.network.neutron [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1801.715055] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1801.738780] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1801.768150] env[62506]: DEBUG nova.network.neutron [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1801.840793] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1801.898624] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1801.934760] env[62506]: DEBUG nova.network.neutron [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1802.110729] env[62506]: INFO nova.compute.manager [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] instance snapshotting [ 1802.110729] env[62506]: DEBUG nova.objects.instance [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lazy-loading 'flavor' on Instance uuid 3aca933d-f1e7-4dbe-a487-f8954da24981 {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 1802.241087] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1802.274397] env[62506]: DEBUG oslo_concurrency.lockutils [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] Releasing lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1802.275021] env[62506]: DEBUG nova.compute.manager [req-433fbcdd-a25f-4018-ab63-46fe335c870a req-3cca08d2-a094-42e5-926f-5c6e1dcbf63f service nova] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Received event network-vif-deleted-f3c5e5d1-9872-4613-9745-72ee1b87647d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1802.363917] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1802.383143] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2463d92c-1e46-49c0-8c66-115ab220b93f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1802.391233] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cffec8c1-337b-42a6-80d6-bf3580c9d029 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1802.424740] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Releasing lock "refresh_cache-fb36ae87-2f9c-4973-8c0c-34c4446365cc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1802.424947] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1802.425190] env[62506]: DEBUG nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1802.425331] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1802.428123] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee15321c-6a1c-493c-9836-0b732e8f619c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1802.436266] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bac10d83-230b-4b09-aadb-d675c50f4641 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1802.441384] env[62506]: DEBUG oslo_concurrency.lockutils [req-d554431e-07c3-4451-b241-689f12ed3050 req-113f7b52-1f09-45b1-a4f7-25f116ca36d3 service nova] Releasing lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1802.452457] env[62506]: DEBUG nova.compute.provider_tree [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1802.499647] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1802.613644] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1802.617640] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbffaaa5-82fb-49d7-acdc-76b17222a256 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1802.636570] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53758317-d511-4245-93b5-03a1efd10d11 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1802.955738] env[62506]: DEBUG nova.scheduler.client.report [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1803.004353] env[62506]: DEBUG nova.network.neutron [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1803.105311] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquiring lock "4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1803.105375] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Lock "4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1803.149798] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Creating Snapshot of the VM instance {{(pid=62506) _create_vm_snapshot /opt/stack/nova/nova/virt/vmwareapi/vmops.py:913}} [ 1803.150216] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.CreateSnapshot_Task with opID=oslo.vmware-864e59ed-8f9e-4917-9c19-0e8e68add4f8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1803.160662] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1803.160662] env[62506]: value = "task-2190777" [ 1803.160662] env[62506]: _type = "Task" [ 1803.160662] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1803.172459] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190777, 'name': CreateSnapshot_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1803.462062] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.279s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1803.462605] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1803.466909] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.135s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1803.507116] env[62506]: INFO nova.compute.manager [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: fb36ae87-2f9c-4973-8c0c-34c4446365cc] Took 1.08 seconds to deallocate network for instance. [ 1803.608808] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1803.627242] env[62506]: ERROR nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1803.627242] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1803.627242] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1803.627242] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1803.627242] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1803.627242] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1803.627242] env[62506]: ERROR nova.compute.manager raise self.value [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1803.627242] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1803.627242] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1803.627242] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1803.627827] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1803.627827] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1803.627827] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1803.627827] env[62506]: ERROR nova.compute.manager [ 1803.627827] env[62506]: Traceback (most recent call last): [ 1803.627827] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1803.627827] env[62506]: listener.cb(fileno) [ 1803.627827] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1803.627827] env[62506]: result = function(*args, **kwargs) [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1803.627827] env[62506]: return func(*args, **kwargs) [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1803.627827] env[62506]: raise e [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1803.627827] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1803.627827] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1803.627827] env[62506]: with excutils.save_and_reraise_exception(): [ 1803.627827] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1803.627827] env[62506]: self.force_reraise() [ 1803.627827] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1803.627827] env[62506]: raise self.value [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1803.627827] env[62506]: updated_port = self._update_port( [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1803.627827] env[62506]: _ensure_no_port_binding_failure(port) [ 1803.627827] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1803.627827] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1803.628728] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1803.628728] env[62506]: Removing descriptor: 15 [ 1803.628728] env[62506]: ERROR nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Traceback (most recent call last): [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] yield resources [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self.driver.spawn(context, instance, image_meta, [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1803.628728] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] vm_ref = self.build_virtual_machine(instance, [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] vif_infos = vmwarevif.get_vif_info(self._session, [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] for vif in network_info: [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return self._sync_wrapper(fn, *args, **kwargs) [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self.wait() [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self[:] = self._gt.wait() [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return self._exit_event.wait() [ 1803.629130] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] result = hub.switch() [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return self.greenlet.switch() [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] result = function(*args, **kwargs) [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return func(*args, **kwargs) [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] raise e [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] nwinfo = self.network_api.allocate_for_instance( [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1803.629511] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] created_port_ids = self._update_ports_for_instance( [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] with excutils.save_and_reraise_exception(): [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self.force_reraise() [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] raise self.value [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] updated_port = self._update_port( [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] _ensure_no_port_binding_failure(port) [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1803.629907] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] raise exception.PortBindingFailed(port_id=port['id']) [ 1803.630268] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1803.630268] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] [ 1803.630268] env[62506]: INFO nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Terminating instance [ 1803.648821] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquiring lock "cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1803.649114] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Lock "cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1803.672965] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190777, 'name': CreateSnapshot_Task, 'duration_secs': 0.481054} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1803.673233] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Created Snapshot of the VM instance {{(pid=62506) _create_vm_snapshot /opt/stack/nova/nova/virt/vmwareapi/vmops.py:922}} [ 1803.676257] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b461ba06-322d-41d1-b1c2-0165464686e2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1803.977152] env[62506]: DEBUG nova.compute.utils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1803.985305] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1803.985511] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1804.133183] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1804.133785] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1804.133859] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquired lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1804.137184] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1804.200513] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Creating linked-clone VM from snapshot {{(pid=62506) _create_linked_clone_from_snapshot /opt/stack/nova/nova/virt/vmwareapi/vmops.py:955}} [ 1804.200513] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.CloneVM_Task with opID=oslo.vmware-3f100513-f6d4-4005-99e1-1218471522a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1804.213401] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1804.213401] env[62506]: value = "task-2190778" [ 1804.213401] env[62506]: _type = "Task" [ 1804.213401] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1804.219738] env[62506]: DEBUG nova.policy [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0611fd263c1448efa03c58d6ce44e8a0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'b7bc3bb2a3cf4158a62b20372a9d7e8e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1804.222813] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1cb25a0-c472-4356-b4b2-e027895da1e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1804.231869] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190778, 'name': CloneVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1804.235114] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fade1e9-9cd0-4173-8247-41497ccc3fc2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1804.273757] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb48c8af-db5c-4b4a-9587-394eabb6e0d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1804.283830] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7cce12b-e98f-4614-b1b4-3042141d0db3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1804.300565] env[62506]: DEBUG nova.compute.provider_tree [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1804.486184] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1804.562735] env[62506]: INFO nova.scheduler.client.report [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Deleted allocations for instance fb36ae87-2f9c-4973-8c0c-34c4446365cc [ 1804.714540] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1804.731146] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190778, 'name': CloneVM_Task} progress is 94%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1804.740939] env[62506]: ERROR nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1804.740939] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1804.740939] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1804.740939] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1804.740939] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1804.740939] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1804.740939] env[62506]: ERROR nova.compute.manager raise self.value [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1804.740939] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1804.740939] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1804.740939] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1804.741646] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1804.741646] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1804.741646] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1804.741646] env[62506]: ERROR nova.compute.manager [ 1804.741646] env[62506]: Traceback (most recent call last): [ 1804.741646] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1804.741646] env[62506]: listener.cb(fileno) [ 1804.741646] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1804.741646] env[62506]: result = function(*args, **kwargs) [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1804.741646] env[62506]: return func(*args, **kwargs) [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1804.741646] env[62506]: raise e [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1804.741646] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1804.741646] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1804.741646] env[62506]: with excutils.save_and_reraise_exception(): [ 1804.741646] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1804.741646] env[62506]: self.force_reraise() [ 1804.741646] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1804.741646] env[62506]: raise self.value [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1804.741646] env[62506]: updated_port = self._update_port( [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1804.741646] env[62506]: _ensure_no_port_binding_failure(port) [ 1804.741646] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1804.741646] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1804.742384] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1804.742384] env[62506]: Removing descriptor: 17 [ 1804.742384] env[62506]: ERROR nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Traceback (most recent call last): [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] yield resources [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self.driver.spawn(context, instance, image_meta, [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1804.742384] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] vm_ref = self.build_virtual_machine(instance, [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] vif_infos = vmwarevif.get_vif_info(self._session, [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] for vif in network_info: [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return self._sync_wrapper(fn, *args, **kwargs) [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self.wait() [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self[:] = self._gt.wait() [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return self._exit_event.wait() [ 1804.742691] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] result = hub.switch() [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return self.greenlet.switch() [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] result = function(*args, **kwargs) [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return func(*args, **kwargs) [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] raise e [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] nwinfo = self.network_api.allocate_for_instance( [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1804.743656] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] created_port_ids = self._update_ports_for_instance( [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] with excutils.save_and_reraise_exception(): [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self.force_reraise() [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] raise self.value [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] updated_port = self._update_port( [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] _ensure_no_port_binding_failure(port) [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1804.744072] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] raise exception.PortBindingFailed(port_id=port['id']) [ 1804.745653] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1804.745653] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] [ 1804.745653] env[62506]: INFO nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Terminating instance [ 1804.806021] env[62506]: DEBUG nova.scheduler.client.report [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1805.020104] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquiring lock "4a95ce6c-d1c0-4677-a330-70620a05989d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1805.020366] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Lock "4a95ce6c-d1c0-4677-a330-70620a05989d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1805.052170] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1805.071332] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0825d332-fbcb-48e8-be18-6507c19ee332 tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "fb36ae87-2f9c-4973-8c0c-34c4446365cc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 21.235s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1805.228585] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190778, 'name': CloneVM_Task} progress is 95%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1805.251324] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1805.251578] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquired lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1805.254766] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1805.314646] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.848s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1805.315723] env[62506]: ERROR nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Traceback (most recent call last): [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self.driver.spawn(context, instance, image_meta, [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] vm_ref = self.build_virtual_machine(instance, [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1805.315723] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] for vif in network_info: [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return self._sync_wrapper(fn, *args, **kwargs) [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self.wait() [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self[:] = self._gt.wait() [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return self._exit_event.wait() [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] result = hub.switch() [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1805.316097] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return self.greenlet.switch() [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] result = function(*args, **kwargs) [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] return func(*args, **kwargs) [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] raise e [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] nwinfo = self.network_api.allocate_for_instance( [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] created_port_ids = self._update_ports_for_instance( [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] with excutils.save_and_reraise_exception(): [ 1805.316532] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] self.force_reraise() [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] raise self.value [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] updated_port = self._update_port( [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] _ensure_no_port_binding_failure(port) [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] raise exception.PortBindingFailed(port_id=port['id']) [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] nova.exception.PortBindingFailed: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. [ 1805.316877] env[62506]: ERROR nova.compute.manager [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] [ 1805.317313] env[62506]: DEBUG nova.compute.utils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1805.318303] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 5.903s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1805.322431] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Build of instance 05b2a777-cf0c-4fdf-a024-6949bd016f6f was re-scheduled: Binding failed for port 1768d8b1-22ff-4739-90ab-8802f299cd38, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1805.322888] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1805.323195] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1805.323334] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1805.323496] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1805.377274] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Successfully created port: 8c510593-5dc7-4c66-860a-98fed06ca332 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1805.416106] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "3aca933d-f1e7-4dbe-a487-f8954da24981" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1805.417024] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "3aca933d-f1e7-4dbe-a487-f8954da24981" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1805.417024] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "3aca933d-f1e7-4dbe-a487-f8954da24981-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1805.418116] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "3aca933d-f1e7-4dbe-a487-f8954da24981-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1805.418116] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "3aca933d-f1e7-4dbe-a487-f8954da24981-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1805.420743] env[62506]: INFO nova.compute.manager [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Terminating instance [ 1805.503343] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1805.537853] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1805.538112] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1805.538268] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1805.538445] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1805.538584] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1805.538723] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1805.538922] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1805.540612] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1805.542400] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1805.542400] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1805.542400] env[62506]: DEBUG nova.virt.hardware [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1805.542614] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13b6e548-8ed1-4efa-8fee-4b6e3b5903b1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.555384] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1a1645e-d868-418c-afe9-94f08f0e4dd2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.563596] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Releasing lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1805.563596] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1805.563596] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1805.563596] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8e4fd8c0-c3f7-424b-b39e-4472b322f82f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.578678] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1805.590051] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3e6424f-3dcd-4ac0-915b-eb2374805fd2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.620720] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4bac9b51-a7cb-4499-be0d-e404039df5f0 could not be found. [ 1805.624341] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1805.624656] env[62506]: INFO nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1805.625285] env[62506]: DEBUG oslo.service.loopingcall [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1805.625285] env[62506]: DEBUG nova.compute.manager [-] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1805.625285] env[62506]: DEBUG nova.network.neutron [-] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1805.731797] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190778, 'name': CloneVM_Task, 'duration_secs': 1.138928} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1805.732365] env[62506]: INFO nova.virt.vmwareapi.vmops [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Created linked-clone VM from snapshot [ 1805.733222] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94c008e0-47c9-43fd-b2a3-36223ac0cd00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.742117] env[62506]: DEBUG nova.virt.vmwareapi.images [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Uploading image 9756a86e-9e04-4a0e-b8ec-d5cbed4fac0d {{(pid=62506) upload_image_stream_optimized /opt/stack/nova/nova/virt/vmwareapi/images.py:441}} [ 1805.753524] env[62506]: DEBUG nova.network.neutron [-] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1805.779249] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Creating HttpNfcLease lease for exporting VM: (result){ [ 1805.779249] env[62506]: value = "vm-446958" [ 1805.779249] env[62506]: _type = "VirtualMachine" [ 1805.779249] env[62506]: }. {{(pid=62506) _create_export_vm_lease /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:478}} [ 1805.780878] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ExportVm with opID=oslo.vmware-8f79538a-7338-4831-bd6f-640cc11397ce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.792049] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lease: (returnval){ [ 1805.792049] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c9b743-cd40-2607-8ea3-aae0dc6eb934" [ 1805.792049] env[62506]: _type = "HttpNfcLease" [ 1805.792049] env[62506]: } obtained for exporting VM: (result){ [ 1805.792049] env[62506]: value = "vm-446958" [ 1805.792049] env[62506]: _type = "VirtualMachine" [ 1805.792049] env[62506]: }. {{(pid=62506) _create_export_vm_lease /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:481}} [ 1805.792355] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the lease: (returnval){ [ 1805.792355] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c9b743-cd40-2607-8ea3-aae0dc6eb934" [ 1805.792355] env[62506]: _type = "HttpNfcLease" [ 1805.792355] env[62506]: } to be ready. {{(pid=62506) wait_for_lease_ready /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:462}} [ 1805.803980] env[62506]: DEBUG oslo_vmware.api [-] Lease: (returnval){ [ 1805.803980] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c9b743-cd40-2607-8ea3-aae0dc6eb934" [ 1805.803980] env[62506]: _type = "HttpNfcLease" [ 1805.803980] env[62506]: } is ready. {{(pid=62506) _poll_lease /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:487}} [ 1805.803980] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Invoking VIM API for reading info of lease: (returnval){ [ 1805.803980] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c9b743-cd40-2607-8ea3-aae0dc6eb934" [ 1805.803980] env[62506]: _type = "HttpNfcLease" [ 1805.803980] env[62506]: }. {{(pid=62506) _create_export_vm_lease /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:486}} [ 1805.804280] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b8d727e-f687-4829-982c-6bca08332f8d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.813041] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Found VMDK URL: https://esx7c1n1.openstack.eu-de-1.cloud.sap/nfc/525d4c1f-669d-577a-6fce-29c3f2067403/disk-0.vmdk from lease info. {{(pid=62506) _find_vmdk_url /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:531}} [ 1805.813259] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Opening URL: https://esx7c1n1.openstack.eu-de-1.cloud.sap/nfc/525d4c1f-669d-577a-6fce-29c3f2067403/disk-0.vmdk for reading. {{(pid=62506) _create_read_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:99}} [ 1805.886091] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1805.892408] env[62506]: DEBUG nova.compute.manager [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Received event network-changed-2c1994e5-967d-42ea-932c-c787807208fe {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1805.892408] env[62506]: DEBUG nova.compute.manager [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Refreshing instance network info cache due to event network-changed-2c1994e5-967d-42ea-932c-c787807208fe. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1805.892408] env[62506]: DEBUG oslo_concurrency.lockutils [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] Acquiring lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1805.892408] env[62506]: DEBUG oslo_concurrency.lockutils [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] Acquired lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1805.892408] env[62506]: DEBUG nova.network.neutron [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Refreshing network info cache for port 2c1994e5-967d-42ea-932c-c787807208fe {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1805.926140] env[62506]: DEBUG oslo_vmware.service [-] Invoking HttpNfcLease.HttpNfcLeaseProgress with opID=oslo.vmware-dec331d8-113b-4530-8209-42b83563d425 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1805.929564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "refresh_cache-3aca933d-f1e7-4dbe-a487-f8954da24981" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1805.929783] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquired lock "refresh_cache-3aca933d-f1e7-4dbe-a487-f8954da24981" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1805.930049] env[62506]: DEBUG nova.network.neutron [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1805.961207] env[62506]: DEBUG nova.compute.manager [req-f1521da4-7fe7-4144-97ff-8d6b473b6f35 req-f37cb571-045b-41d9-9949-778127c22bd9 service nova] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Received event network-vif-deleted-7f21cb96-831a-42cb-890e-8cefb13ac7a5 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1805.989906] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1806.111093] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1806.161419] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-516877c6-317b-49a1-8a93-b694ecf0770e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1806.173778] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46ee1923-32e8-431d-b255-6adae7c1a660 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1806.207875] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de4fa23a-8d45-40fd-8ea9-47c26810d567 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1806.218713] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d21f00a-f74f-44c4-8b88-f41fa3e6174e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1806.234823] env[62506]: DEBUG nova.compute.provider_tree [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1806.258426] env[62506]: DEBUG nova.network.neutron [-] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1806.491179] env[62506]: DEBUG nova.network.neutron [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1806.497026] env[62506]: DEBUG nova.network.neutron [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1806.611342] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1806.611342] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1806.611342] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1806.632461] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1806.649024] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1806.692598] env[62506]: DEBUG nova.network.neutron [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1806.737959] env[62506]: DEBUG nova.scheduler.client.report [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1806.763331] env[62506]: INFO nova.compute.manager [-] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Took 1.14 seconds to deallocate network for instance. [ 1806.766527] env[62506]: DEBUG nova.compute.claims [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1806.766724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1806.796321] env[62506]: DEBUG nova.network.neutron [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1807.116792] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1807.135749] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Releasing lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1807.136223] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1807.136421] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1807.136746] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bf27826a-acec-4777-99bb-bb91d7a3c98f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1807.146458] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2918a468-5ee4-4409-8f69-1c164c919c9d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1807.157887] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-05b2a777-cf0c-4fdf-a024-6949bd016f6f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1807.158341] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1807.158493] env[62506]: DEBUG nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1807.158661] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1807.174033] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa could not be found. [ 1807.174287] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1807.174476] env[62506]: INFO nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1807.174735] env[62506]: DEBUG oslo.service.loopingcall [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1807.175081] env[62506]: DEBUG nova.compute.manager [-] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1807.175234] env[62506]: DEBUG nova.network.neutron [-] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1807.196475] env[62506]: DEBUG oslo_concurrency.lockutils [req-80fb283e-7510-4195-85a4-07cf75460e94 req-1650528b-d11b-475a-810e-0fa75ec18392 service nova] Releasing lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1807.212514] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1807.227772] env[62506]: DEBUG nova.network.neutron [-] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1807.245732] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.927s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1807.246459] env[62506]: ERROR nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Traceback (most recent call last): [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self.driver.spawn(context, instance, image_meta, [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] vm_ref = self.build_virtual_machine(instance, [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] vif_infos = vmwarevif.get_vif_info(self._session, [ 1807.246459] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] for vif in network_info: [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return self._sync_wrapper(fn, *args, **kwargs) [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self.wait() [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self[:] = self._gt.wait() [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return self._exit_event.wait() [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] result = hub.switch() [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1807.246939] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return self.greenlet.switch() [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] result = function(*args, **kwargs) [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] return func(*args, **kwargs) [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] raise e [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] nwinfo = self.network_api.allocate_for_instance( [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] created_port_ids = self._update_ports_for_instance( [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] with excutils.save_and_reraise_exception(): [ 1807.247481] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] self.force_reraise() [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] raise self.value [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] updated_port = self._update_port( [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] _ensure_no_port_binding_failure(port) [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] raise exception.PortBindingFailed(port_id=port['id']) [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] nova.exception.PortBindingFailed: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. [ 1807.248087] env[62506]: ERROR nova.compute.manager [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] [ 1807.248827] env[62506]: DEBUG nova.compute.utils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1807.248827] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.099s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1807.252816] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Build of instance edc70971-dcfe-4575-9af8-d31e77d1bcf1 was re-scheduled: Binding failed for port f3c5e5d1-9872-4613-9745-72ee1b87647d, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1807.255262] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1807.255262] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquiring lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1807.255262] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Acquired lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1807.255262] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1807.298147] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Releasing lock "refresh_cache-3aca933d-f1e7-4dbe-a487-f8954da24981" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1807.299245] env[62506]: DEBUG nova.compute.manager [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1807.300153] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1807.301153] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61e25438-9350-4fc2-a585-bddcd7e45912 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1807.311509] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 1807.311777] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-3238e1a2-5e1f-46d8-9ac2-8e41452d7744 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1807.322019] env[62506]: DEBUG oslo_vmware.api [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1807.322019] env[62506]: value = "task-2190780" [ 1807.322019] env[62506]: _type = "Task" [ 1807.322019] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1807.334124] env[62506]: DEBUG oslo_vmware.api [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190780, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1807.720140] env[62506]: DEBUG nova.network.neutron [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1807.733036] env[62506]: DEBUG nova.network.neutron [-] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1807.797601] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1807.836150] env[62506]: DEBUG oslo_vmware.api [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190780, 'name': PowerOffVM_Task, 'duration_secs': 0.222609} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1807.839335] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 1807.839721] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 1807.843033] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d3b39806-e15f-4343-bb68-86f767a19894 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1807.867527] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 1807.868435] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 1807.868435] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Deleting the datastore file [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1807.868435] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-07e9ee68-b1fd-4aa9-8b38-c64e8e9020d0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1807.876660] env[62506]: DEBUG oslo_vmware.api [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1807.876660] env[62506]: value = "task-2190782" [ 1807.876660] env[62506]: _type = "Task" [ 1807.876660] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1807.892642] env[62506]: DEBUG oslo_vmware.api [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190782, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1807.939879] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1808.009457] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-135efb95-54ca-49c4-bd34-174bf834e35a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1808.018738] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7962b651-9a58-4b16-84a9-2597225650a3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1808.057524] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9ac04e2-9a5b-4667-8d2c-8f141633406f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1808.072307] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e79a36cf-f1d7-442e-bae3-d321b42ea49f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1808.093155] env[62506]: DEBUG nova.compute.provider_tree [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1808.225974] env[62506]: INFO nova.compute.manager [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 05b2a777-cf0c-4fdf-a024-6949bd016f6f] Took 1.07 seconds to deallocate network for instance. [ 1808.238641] env[62506]: INFO nova.compute.manager [-] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Took 1.06 seconds to deallocate network for instance. [ 1808.242879] env[62506]: DEBUG nova.compute.claims [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1808.243073] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1808.388285] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] In vmwareapi:vmops:_destroy_instance, exception while deleting the VM contents from the disk: oslo_vmware.exceptions.CannotDeleteFileException: Cannot delete file [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981 [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Traceback (most recent call last): [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 1113, in _destroy_instance [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] ds_util.file_delete(self._session, [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/nova/nova/virt/vmwareapi/ds_util.py", line 219, in file_delete [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] session._wait_for_task(file_delete_task) [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] return self.wait_for_task(task_ref) [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] return evt.wait() [ 1808.388285] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] result = hub.switch() [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] return self.greenlet.switch() [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] self.f(*self.args, **self.kw) [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] raise exceptions.translate_fault(task_info.error) [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] oslo_vmware.exceptions.CannotDeleteFileException: Cannot delete file [datastore2] 3aca933d-f1e7-4dbe-a487-f8954da24981 [ 1808.388633] env[62506]: ERROR nova.virt.vmwareapi.vmops [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] [ 1808.388633] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1808.388927] env[62506]: INFO nova.compute.manager [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Took 1.09 seconds to destroy the instance on the hypervisor. [ 1808.388927] env[62506]: DEBUG oslo.service.loopingcall [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1808.388927] env[62506]: DEBUG nova.compute.manager [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1808.388927] env[62506]: DEBUG nova.network.neutron [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1808.412091] env[62506]: DEBUG nova.network.neutron [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1808.450937] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Releasing lock "refresh_cache-edc70971-dcfe-4575-9af8-d31e77d1bcf1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1808.450937] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1808.450937] env[62506]: DEBUG nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1808.450937] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1808.469072] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1808.597441] env[62506]: DEBUG nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1808.638047] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "b5527c82-048e-4738-8878-0807aa609596" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1808.638295] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "b5527c82-048e-4738-8878-0807aa609596" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1808.917207] env[62506]: DEBUG nova.network.neutron [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1808.954476] env[62506]: ERROR nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1808.954476] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1808.954476] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1808.954476] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1808.954476] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1808.954476] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1808.954476] env[62506]: ERROR nova.compute.manager raise self.value [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1808.954476] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1808.954476] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1808.954476] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1808.956437] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1808.956437] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1808.956437] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1808.956437] env[62506]: ERROR nova.compute.manager [ 1808.956437] env[62506]: Traceback (most recent call last): [ 1808.956437] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1808.956437] env[62506]: listener.cb(fileno) [ 1808.956437] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1808.956437] env[62506]: result = function(*args, **kwargs) [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1808.956437] env[62506]: return func(*args, **kwargs) [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1808.956437] env[62506]: raise e [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1808.956437] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1808.956437] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1808.956437] env[62506]: with excutils.save_and_reraise_exception(): [ 1808.956437] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1808.956437] env[62506]: self.force_reraise() [ 1808.956437] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1808.956437] env[62506]: raise self.value [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1808.956437] env[62506]: updated_port = self._update_port( [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1808.956437] env[62506]: _ensure_no_port_binding_failure(port) [ 1808.956437] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1808.956437] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1808.957313] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1808.957313] env[62506]: Removing descriptor: 15 [ 1808.957313] env[62506]: ERROR nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Traceback (most recent call last): [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] yield resources [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self.driver.spawn(context, instance, image_meta, [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1808.957313] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] vm_ref = self.build_virtual_machine(instance, [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] for vif in network_info: [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return self._sync_wrapper(fn, *args, **kwargs) [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self.wait() [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self[:] = self._gt.wait() [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return self._exit_event.wait() [ 1808.957715] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] result = hub.switch() [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return self.greenlet.switch() [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] result = function(*args, **kwargs) [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return func(*args, **kwargs) [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] raise e [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] nwinfo = self.network_api.allocate_for_instance( [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1808.958114] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] created_port_ids = self._update_ports_for_instance( [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] with excutils.save_and_reraise_exception(): [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self.force_reraise() [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] raise self.value [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] updated_port = self._update_port( [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] _ensure_no_port_binding_failure(port) [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1808.958510] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] raise exception.PortBindingFailed(port_id=port['id']) [ 1808.958909] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1808.958909] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] [ 1808.958909] env[62506]: INFO nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Terminating instance [ 1808.974290] env[62506]: DEBUG nova.network.neutron [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1809.075919] env[62506]: DEBUG nova.compute.manager [req-fd5938d2-8970-497d-a255-b31aaa0172de req-c178cea7-bcf2-4f77-b4cf-12dda80a87b1 service nova] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Received event network-vif-deleted-2c1994e5-967d-42ea-932c-c787807208fe {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1809.104421] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.856s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1809.105071] env[62506]: ERROR nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] Traceback (most recent call last): [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self.driver.spawn(context, instance, image_meta, [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] vm_ref = self.build_virtual_machine(instance, [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] vif_infos = vmwarevif.get_vif_info(self._session, [ 1809.105071] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] for vif in network_info: [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return self._sync_wrapper(fn, *args, **kwargs) [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self.wait() [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self[:] = self._gt.wait() [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return self._exit_event.wait() [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] result = hub.switch() [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1809.105631] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return self.greenlet.switch() [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] result = function(*args, **kwargs) [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] return func(*args, **kwargs) [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] raise e [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] nwinfo = self.network_api.allocate_for_instance( [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] created_port_ids = self._update_ports_for_instance( [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] with excutils.save_and_reraise_exception(): [ 1809.107044] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] self.force_reraise() [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] raise self.value [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] updated_port = self._update_port( [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] _ensure_no_port_binding_failure(port) [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] raise exception.PortBindingFailed(port_id=port['id']) [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] nova.exception.PortBindingFailed: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. [ 1809.107533] env[62506]: ERROR nova.compute.manager [instance: ca66e019-db83-4510-a049-8e8147537fa5] [ 1809.107883] env[62506]: DEBUG nova.compute.utils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1809.108836] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.867s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1809.111434] env[62506]: INFO nova.compute.claims [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1809.114919] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Build of instance ca66e019-db83-4510-a049-8e8147537fa5 was re-scheduled: Binding failed for port 7f21cb96-831a-42cb-890e-8cefb13ac7a5, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1809.116370] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1809.116624] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquiring lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1809.116787] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Acquired lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1809.117308] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1809.272381] env[62506]: INFO nova.scheduler.client.report [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Deleted allocations for instance 05b2a777-cf0c-4fdf-a024-6949bd016f6f [ 1809.280684] env[62506]: DEBUG nova.compute.manager [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Received event network-changed-d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1809.281024] env[62506]: DEBUG nova.compute.manager [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Refreshing instance network info cache due to event network-changed-d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1809.281441] env[62506]: DEBUG oslo_concurrency.lockutils [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] Acquiring lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1809.281754] env[62506]: DEBUG oslo_concurrency.lockutils [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] Acquired lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1809.282329] env[62506]: DEBUG nova.network.neutron [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Refreshing network info cache for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1809.420497] env[62506]: INFO nova.compute.manager [-] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Took 1.03 seconds to deallocate network for instance. [ 1809.466310] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquiring lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1809.466310] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquired lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1809.466310] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1809.479269] env[62506]: INFO nova.compute.manager [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] [instance: edc70971-dcfe-4575-9af8-d31e77d1bcf1] Took 1.03 seconds to deallocate network for instance. [ 1809.659185] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1809.786729] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5d93b377-3cc7-46a4-85ae-e537f8283170 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "05b2a777-cf0c-4fdf-a024-6949bd016f6f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 23.931s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1809.842213] env[62506]: DEBUG nova.network.neutron [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1809.929571] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1810.051923] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1810.154464] env[62506]: DEBUG nova.network.neutron [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1810.183891] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1810.297130] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1810.357350] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1810.397652] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-801d753a-f9ec-4409-af48-b31a10b8e488 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.407958] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54d18edc-2630-45f8-aa49-10df61708654 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.448108] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e799ae74-058f-422c-8327-0669f93d777e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.457774] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ea92083-0623-4257-be55-9ae701ca2c39 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.477586] env[62506]: DEBUG nova.compute.provider_tree [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1810.527558] env[62506]: INFO nova.scheduler.client.report [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Deleted allocations for instance edc70971-dcfe-4575-9af8-d31e77d1bcf1 [ 1810.659398] env[62506]: DEBUG oslo_concurrency.lockutils [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] Releasing lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1810.659667] env[62506]: DEBUG nova.compute.manager [req-6b9f8a0d-b9b5-4695-9fcd-81f8d3d01630 req-f0c2b55e-19ba-46d1-96f5-544b62986538 service nova] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Received event network-vif-deleted-d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1810.686789] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Releasing lock "refresh_cache-ca66e019-db83-4510-a049-8e8147537fa5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1810.686789] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1810.686950] env[62506]: DEBUG nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1810.687349] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1810.718490] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1810.829194] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1810.863485] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Releasing lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1810.864141] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1810.865140] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1810.865140] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5adf0ddc-4952-4235-9ee7-60d967c0bc4c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.875251] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4612cfac-6428-4108-bb6f-2640f4d4caea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.901712] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 54e34c6f-3863-4683-b301-a6a9d25f1c4d could not be found. [ 1810.901841] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1810.902139] env[62506]: INFO nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1810.902540] env[62506]: DEBUG oslo.service.loopingcall [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1810.902802] env[62506]: DEBUG nova.compute.manager [-] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1810.902900] env[62506]: DEBUG nova.network.neutron [-] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1810.943051] env[62506]: DEBUG nova.network.neutron [-] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1810.981877] env[62506]: DEBUG nova.scheduler.client.report [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1811.039310] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6ef4487d-239f-4f77-b629-9ad22d2aba33 tempest-FloatingIPsAssociationTestJSON-2087750739 tempest-FloatingIPsAssociationTestJSON-2087750739-project-member] Lock "edc70971-dcfe-4575-9af8-d31e77d1bcf1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 24.526s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.087815] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquiring lock "e433b5a0-2742-4897-a7d9-46571bcc0f2c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1811.087815] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Lock "e433b5a0-2742-4897-a7d9-46571bcc0f2c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.222582] env[62506]: DEBUG nova.network.neutron [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1811.445966] env[62506]: DEBUG nova.network.neutron [-] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1811.488796] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.381s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.489450] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1811.492821] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.129s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.494438] env[62506]: INFO nova.compute.claims [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1811.544593] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1811.725552] env[62506]: INFO nova.compute.manager [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] [instance: ca66e019-db83-4510-a049-8e8147537fa5] Took 1.04 seconds to deallocate network for instance. [ 1811.879355] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquiring lock "386935e4-c2b7-4207-9670-cd532ff12f70" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1811.879558] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Lock "386935e4-c2b7-4207-9670-cd532ff12f70" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.883966] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "af042cab-9132-4425-b017-133ec85afa0c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1811.884277] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "af042cab-9132-4425-b017-133ec85afa0c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.950985] env[62506]: INFO nova.compute.manager [-] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Took 1.05 seconds to deallocate network for instance. [ 1811.953555] env[62506]: DEBUG nova.compute.claims [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1811.953745] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1812.000054] env[62506]: DEBUG nova.compute.utils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1812.009707] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1812.009812] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1812.077790] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1812.111687] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "88322331-7cb9-46a9-8e8c-82f0198c90d8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1812.112065] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "88322331-7cb9-46a9-8e8c-82f0198c90d8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1812.145286] env[62506]: DEBUG nova.policy [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c5df72749d1b47a988c4d196bf770531', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c1e5d6924df748aa81964764f8127f4a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1812.511491] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1812.605966] env[62506]: DEBUG nova.compute.manager [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Received event network-changed-8c510593-5dc7-4c66-860a-98fed06ca332 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1812.605966] env[62506]: DEBUG nova.compute.manager [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Refreshing instance network info cache due to event network-changed-8c510593-5dc7-4c66-860a-98fed06ca332. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1812.605966] env[62506]: DEBUG oslo_concurrency.lockutils [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] Acquiring lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1812.605966] env[62506]: DEBUG oslo_concurrency.lockutils [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] Acquired lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1812.605966] env[62506]: DEBUG nova.network.neutron [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Refreshing network info cache for port 8c510593-5dc7-4c66-860a-98fed06ca332 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1812.794362] env[62506]: INFO nova.scheduler.client.report [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Deleted allocations for instance ca66e019-db83-4510-a049-8e8147537fa5 [ 1812.852478] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d7a3ca6-abf1-4590-a5cd-884d10e8a65b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1812.861968] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d72294ad-9348-4487-872f-d0b421c893ab {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1812.897035] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ad0336a-23ca-47e8-a678-833b40022dd1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1812.907284] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7ba2836-bd76-45a2-9207-29d9e1ca1924 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1812.922889] env[62506]: DEBUG nova.compute.provider_tree [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1813.142463] env[62506]: DEBUG nova.network.neutron [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1813.281718] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Getting lease state for https://esx7c1n1.openstack.eu-de-1.cloud.sap/nfc/525d4c1f-669d-577a-6fce-29c3f2067403/disk-0.vmdk. {{(pid=62506) _release_lease /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:423}} [ 1813.283458] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f8dd84a-3f22-4a41-9550-66007802c9dd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.290666] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lease for https://esx7c1n1.openstack.eu-de-1.cloud.sap/nfc/525d4c1f-669d-577a-6fce-29c3f2067403/disk-0.vmdk is in state: ready. {{(pid=62506) _release_lease /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:430}} [ 1813.291049] env[62506]: ERROR oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Aborting lease for https://esx7c1n1.openstack.eu-de-1.cloud.sap/nfc/525d4c1f-669d-577a-6fce-29c3f2067403/disk-0.vmdk due to incomplete transfer. [ 1813.291390] env[62506]: DEBUG oslo_vmware.service [-] Invoking HttpNfcLease.HttpNfcLeaseAbort with opID=oslo.vmware-8267983a-a23d-4664-81bf-fa561ddba448 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.299522] env[62506]: DEBUG oslo_vmware.rw_handles [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Closed VMDK read handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap/nfc/525d4c1f-669d-577a-6fce-29c3f2067403/disk-0.vmdk. {{(pid=62506) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:735}} [ 1813.299737] env[62506]: DEBUG nova.virt.vmwareapi.images [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Uploaded image 9756a86e-9e04-4a0e-b8ec-d5cbed4fac0d to the Glance image server {{(pid=62506) upload_image_stream_optimized /opt/stack/nova/nova/virt/vmwareapi/images.py:472}} [ 1813.301880] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Destroying the VM {{(pid=62506) destroy_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1368}} [ 1813.301880] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Destroy_Task with opID=oslo.vmware-52782ad1-f44f-4dd1-9e1b-ff4fda08715c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.308041] env[62506]: DEBUG oslo_concurrency.lockutils [None req-27ed7d82-2408-44cf-ab4d-6823ec9a5c8a tempest-ServerActionsTestOtherB-1904199998 tempest-ServerActionsTestOtherB-1904199998-project-member] Lock "ca66e019-db83-4510-a049-8e8147537fa5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 25.447s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1813.310558] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Waiting for the task: (returnval){ [ 1813.310558] env[62506]: value = "task-2190783" [ 1813.310558] env[62506]: _type = "Task" [ 1813.310558] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1813.322074] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190783, 'name': Destroy_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1813.390375] env[62506]: DEBUG nova.network.neutron [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1813.425965] env[62506]: DEBUG nova.scheduler.client.report [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1813.492179] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Successfully created port: 3cf9e11d-bf4f-4884-85f2-3e34051ae260 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1813.526607] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1813.557169] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1813.557442] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1813.557628] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1813.557811] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1813.557956] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1813.558224] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1813.558464] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1813.558624] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1813.558799] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1813.559142] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1813.559354] env[62506]: DEBUG nova.virt.hardware [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1813.561060] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f03d3394-79f8-4628-84ef-cc82bf942629 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.572750] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-381052f7-70c0-4ddc-8a19-2b1bf7096b22 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.812098] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1813.831716] env[62506]: DEBUG oslo_vmware.api [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Task: {'id': task-2190783, 'name': Destroy_Task, 'duration_secs': 0.302086} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1813.831716] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Destroyed the VM [ 1813.831716] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Deleting Snapshot of the VM instance {{(pid=62506) _delete_vm_snapshot /opt/stack/nova/nova/virt/vmwareapi/vmops.py:932}} [ 1813.831716] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachineSnapshot.RemoveSnapshot_Task with opID=oslo.vmware-93e946c1-fa27-497e-a1a6-f4a1592a9434 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.837276] env[62506]: WARNING suds.client [-] Web service reported a SOAP processing fault using an unexpected HTTP status code 200. Reporting as an internal server error. [ 1813.837276] env[62506]: DEBUG oslo_vmware.api [-] Fault list: [ManagedObjectNotFound] {{(pid=62506) _invoke_api /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:337}} [ 1813.837276] env[62506]: DEBUG nova.compute.utils [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Cleaning up image 9756a86e-9e04-4a0e-b8ec-d5cbed4fac0d {{(pid=62506) delete_image /opt/stack/nova/nova/compute/utils.py:1353}} [ 1813.894965] env[62506]: DEBUG oslo_concurrency.lockutils [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] Releasing lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1813.895303] env[62506]: DEBUG nova.compute.manager [req-33793f46-98ae-4a5d-9f6a-96feccad5793 req-e71b8b25-5176-411f-9fb0-fc7678c71224 service nova] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Received event network-vif-deleted-8c510593-5dc7-4c66-860a-98fed06ca332 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1813.932472] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.440s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1813.934028] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1813.943213] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.810s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1813.944836] env[62506]: INFO nova.compute.claims [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1814.341803] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1814.441744] env[62506]: DEBUG nova.compute.utils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1814.443174] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1814.443403] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1814.521172] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquiring lock "6cdfeb65-c53f-4ba6-873c-88e593909878" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1814.521479] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Lock "6cdfeb65-c53f-4ba6-873c-88e593909878" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1814.556891] env[62506]: DEBUG nova.policy [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a65549e701c041debe7591e067bcc129', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '07d447535e8748799f9f7d521db672ba', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1814.951645] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1815.222732] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b41e46ad-cf40-45f7-ac91-4035c9647877 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1815.231384] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd7e5446-9a9c-4eca-a73c-913c90b74a65 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1815.264898] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1bc03c53-dbfd-4780-917a-2274d4eab423 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1815.273106] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7445a1a-e212-435f-96bd-2f6a9a06909b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1815.286944] env[62506]: DEBUG nova.compute.provider_tree [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1815.289451] env[62506]: ERROR nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1815.289451] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1815.289451] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1815.289451] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1815.289451] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1815.289451] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1815.289451] env[62506]: ERROR nova.compute.manager raise self.value [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1815.289451] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1815.289451] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1815.289451] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1815.290078] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1815.290078] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1815.290078] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1815.290078] env[62506]: ERROR nova.compute.manager [ 1815.290078] env[62506]: Traceback (most recent call last): [ 1815.290078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1815.290078] env[62506]: listener.cb(fileno) [ 1815.290078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1815.290078] env[62506]: result = function(*args, **kwargs) [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1815.290078] env[62506]: return func(*args, **kwargs) [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1815.290078] env[62506]: raise e [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1815.290078] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1815.290078] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1815.290078] env[62506]: with excutils.save_and_reraise_exception(): [ 1815.290078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1815.290078] env[62506]: self.force_reraise() [ 1815.290078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1815.290078] env[62506]: raise self.value [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1815.290078] env[62506]: updated_port = self._update_port( [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1815.290078] env[62506]: _ensure_no_port_binding_failure(port) [ 1815.290078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1815.290078] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1815.291423] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1815.291423] env[62506]: Removing descriptor: 19 [ 1815.291423] env[62506]: ERROR nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Traceback (most recent call last): [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] yield resources [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self.driver.spawn(context, instance, image_meta, [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1815.291423] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] vm_ref = self.build_virtual_machine(instance, [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] for vif in network_info: [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return self._sync_wrapper(fn, *args, **kwargs) [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self.wait() [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self[:] = self._gt.wait() [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return self._exit_event.wait() [ 1815.292723] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] result = hub.switch() [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return self.greenlet.switch() [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] result = function(*args, **kwargs) [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return func(*args, **kwargs) [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] raise e [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] nwinfo = self.network_api.allocate_for_instance( [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1815.293743] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] created_port_ids = self._update_ports_for_instance( [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] with excutils.save_and_reraise_exception(): [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self.force_reraise() [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] raise self.value [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] updated_port = self._update_port( [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] _ensure_no_port_binding_failure(port) [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1815.294351] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] raise exception.PortBindingFailed(port_id=port['id']) [ 1815.295414] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1815.295414] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] [ 1815.295414] env[62506]: INFO nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Terminating instance [ 1815.302071] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Successfully created port: 3698a633-4ba4-43b5-8aef-9c4df07c9722 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1815.550728] env[62506]: DEBUG oslo_concurrency.lockutils [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1815.765772] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquiring lock "249e5265-86b3-4ba8-96b9-d21ea0a5599d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1815.766061] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Lock "249e5265-86b3-4ba8-96b9-d21ea0a5599d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1815.793812] env[62506]: DEBUG nova.scheduler.client.report [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1815.799437] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquiring lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1815.799437] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquired lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1815.799437] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1815.843384] env[62506]: DEBUG nova.compute.manager [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Received event network-changed-3cf9e11d-bf4f-4884-85f2-3e34051ae260 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1815.843574] env[62506]: DEBUG nova.compute.manager [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Refreshing instance network info cache due to event network-changed-3cf9e11d-bf4f-4884-85f2-3e34051ae260. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1815.844135] env[62506]: DEBUG oslo_concurrency.lockutils [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] Acquiring lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1815.959916] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1815.987880] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "b4bcab92-0007-4952-8d74-c470049edd71" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1815.988228] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "b4bcab92-0007-4952-8d74-c470049edd71" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1816.003076] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1816.003076] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1816.003287] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1816.004134] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1816.004357] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1816.004853] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1816.005132] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1816.005346] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1816.005558] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1816.005762] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1816.005980] env[62506]: DEBUG nova.virt.hardware [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1816.006923] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-035361f7-c4fe-45dd-84c9-530dc365d271 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1816.020033] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab560348-0033-4998-8864-fb0cdffcb5a7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1816.302279] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquiring lock "a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1816.302509] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Lock "a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1816.305400] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.362s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1816.305816] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1816.308538] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.198s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1816.313408] env[62506]: INFO nova.compute.claims [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1816.500921] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1816.680494] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1816.822289] env[62506]: DEBUG nova.compute.utils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1816.826309] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1816.826993] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1816.964370] env[62506]: DEBUG nova.policy [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cf6915f30d80466f95171eb0d65c1e02', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c7a36fa34db34f37bdb04a558d1e96c0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1817.184288] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Releasing lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1817.184735] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1817.184933] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1817.185266] env[62506]: DEBUG oslo_concurrency.lockutils [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] Acquired lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1817.185437] env[62506]: DEBUG nova.network.neutron [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Refreshing network info cache for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1817.186939] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ec6d30ce-1672-49ad-be96-290e5d35a72e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.197531] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3451af0e-03ce-4186-800b-defce3448c46 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.228284] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c16e39bd-3443-4dbb-8d5a-ab90cf27251f could not be found. [ 1817.228372] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1817.228523] env[62506]: INFO nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1817.228995] env[62506]: DEBUG oslo.service.loopingcall [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1817.228995] env[62506]: DEBUG nova.compute.manager [-] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1817.229125] env[62506]: DEBUG nova.network.neutron [-] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1817.250544] env[62506]: DEBUG nova.network.neutron [-] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1817.328532] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1817.643891] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "8fd213dd-d597-48c0-9387-19b4f5dd72ff" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1817.643891] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "8fd213dd-d597-48c0-9387-19b4f5dd72ff" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1817.662021] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e34c8b7-311e-41b2-9abd-9a4eacbd690b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.672026] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60627a05-29eb-4162-b6e6-e38d64612e43 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.705813] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be7a1b09-13b8-4b5b-8936-ba6127eb0b7e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.715112] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-600d1590-f236-44be-93be-e24696993fc3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.730871] env[62506]: DEBUG nova.compute.provider_tree [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1817.753180] env[62506]: DEBUG nova.network.neutron [-] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1817.858505] env[62506]: DEBUG nova.network.neutron [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1817.954891] env[62506]: DEBUG nova.network.neutron [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1818.024545] env[62506]: ERROR nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1818.024545] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1818.024545] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1818.024545] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1818.024545] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1818.024545] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1818.024545] env[62506]: ERROR nova.compute.manager raise self.value [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1818.024545] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1818.024545] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1818.024545] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1818.025091] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1818.025091] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1818.025091] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1818.025091] env[62506]: ERROR nova.compute.manager [ 1818.025091] env[62506]: Traceback (most recent call last): [ 1818.025091] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1818.025091] env[62506]: listener.cb(fileno) [ 1818.025091] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1818.025091] env[62506]: result = function(*args, **kwargs) [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1818.025091] env[62506]: return func(*args, **kwargs) [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1818.025091] env[62506]: raise e [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1818.025091] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1818.025091] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1818.025091] env[62506]: with excutils.save_and_reraise_exception(): [ 1818.025091] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1818.025091] env[62506]: self.force_reraise() [ 1818.025091] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1818.025091] env[62506]: raise self.value [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1818.025091] env[62506]: updated_port = self._update_port( [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1818.025091] env[62506]: _ensure_no_port_binding_failure(port) [ 1818.025091] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1818.025091] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1818.025962] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1818.025962] env[62506]: Removing descriptor: 15 [ 1818.025962] env[62506]: ERROR nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Traceback (most recent call last): [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] yield resources [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self.driver.spawn(context, instance, image_meta, [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1818.025962] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] vm_ref = self.build_virtual_machine(instance, [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] vif_infos = vmwarevif.get_vif_info(self._session, [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] for vif in network_info: [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return self._sync_wrapper(fn, *args, **kwargs) [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self.wait() [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self[:] = self._gt.wait() [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return self._exit_event.wait() [ 1818.026335] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] result = hub.switch() [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return self.greenlet.switch() [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] result = function(*args, **kwargs) [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return func(*args, **kwargs) [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] raise e [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] nwinfo = self.network_api.allocate_for_instance( [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1818.026707] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] created_port_ids = self._update_ports_for_instance( [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] with excutils.save_and_reraise_exception(): [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self.force_reraise() [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] raise self.value [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] updated_port = self._update_port( [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] _ensure_no_port_binding_failure(port) [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1818.027209] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] raise exception.PortBindingFailed(port_id=port['id']) [ 1818.028536] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1818.028536] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] [ 1818.028536] env[62506]: INFO nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Terminating instance [ 1818.052140] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Successfully created port: 43739195-3ff0-497a-871e-7dd5f4b9a3f9 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1818.236124] env[62506]: DEBUG nova.scheduler.client.report [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1818.253343] env[62506]: DEBUG nova.compute.manager [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Received event network-vif-deleted-3cf9e11d-bf4f-4884-85f2-3e34051ae260 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1818.253343] env[62506]: DEBUG nova.compute.manager [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Received event network-changed-3698a633-4ba4-43b5-8aef-9c4df07c9722 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1818.253505] env[62506]: DEBUG nova.compute.manager [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Refreshing instance network info cache due to event network-changed-3698a633-4ba4-43b5-8aef-9c4df07c9722. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1818.253788] env[62506]: DEBUG oslo_concurrency.lockutils [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] Acquiring lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1818.253850] env[62506]: DEBUG oslo_concurrency.lockutils [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] Acquired lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1818.253992] env[62506]: DEBUG nova.network.neutron [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Refreshing network info cache for port 3698a633-4ba4-43b5-8aef-9c4df07c9722 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1818.257594] env[62506]: INFO nova.compute.manager [-] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Took 1.03 seconds to deallocate network for instance. [ 1818.266309] env[62506]: DEBUG nova.compute.claims [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1818.266309] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1818.342307] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1818.368311] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1818.368585] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1818.368743] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1818.369011] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1818.370042] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1818.370042] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1818.370042] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1818.370042] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1818.370410] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1818.371047] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1818.371047] env[62506]: DEBUG nova.virt.hardware [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1818.371823] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afc5f246-1560-42f3-80f6-3936f632e6cd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1818.383697] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75ec7b37-1f36-45b6-8c87-e2d41654e746 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1818.461232] env[62506]: DEBUG oslo_concurrency.lockutils [req-3cafc6d7-8275-4fb0-9246-b5732e5ab0fb req-9a4a1ef3-0533-45b5-a920-2d509d43de4e service nova] Releasing lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1818.533230] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquiring lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1818.744157] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.435s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1818.744822] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1818.748036] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.981s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1818.820449] env[62506]: DEBUG nova.network.neutron [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1819.219249] env[62506]: DEBUG nova.network.neutron [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1819.255384] env[62506]: DEBUG nova.compute.utils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1819.268434] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1819.268434] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1819.579142] env[62506]: DEBUG nova.policy [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '81d0148faa8e41e8a3bbafa2581aa557', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c47a8a2b3ef54338a39ff4aac0d33170', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1819.597029] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80752f6f-2464-4cbd-84dd-34a92c06a3b8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1819.603319] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-688b40b6-2248-46d7-b110-dbf8b32ca952 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1819.644348] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-515bfb7d-cd3c-4d44-af97-daf76af88d93 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1819.653337] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46c0cfaf-81b4-4601-84e3-b1a66820dd90 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1819.668773] env[62506]: DEBUG nova.compute.provider_tree [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1819.725021] env[62506]: DEBUG oslo_concurrency.lockutils [req-3eceffed-c2f3-4f6d-84c0-30e60e80c171 req-a5ad8cb8-8d0d-4afc-a2c1-c9c26424c0f3 service nova] Releasing lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1819.725450] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquired lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1819.725645] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1819.765824] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1820.174904] env[62506]: DEBUG nova.scheduler.client.report [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1820.254974] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1820.417416] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1820.681784] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.934s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1820.682460] env[62506]: ERROR nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Traceback (most recent call last): [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self.driver.spawn(context, instance, image_meta, [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] vm_ref = self.build_virtual_machine(instance, [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] vif_infos = vmwarevif.get_vif_info(self._session, [ 1820.682460] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] for vif in network_info: [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return self._sync_wrapper(fn, *args, **kwargs) [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self.wait() [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self[:] = self._gt.wait() [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return self._exit_event.wait() [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] result = hub.switch() [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1820.682827] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return self.greenlet.switch() [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] result = function(*args, **kwargs) [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] return func(*args, **kwargs) [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] raise e [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] nwinfo = self.network_api.allocate_for_instance( [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] created_port_ids = self._update_ports_for_instance( [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] with excutils.save_and_reraise_exception(): [ 1820.683448] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] self.force_reraise() [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] raise self.value [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] updated_port = self._update_port( [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] _ensure_no_port_binding_failure(port) [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] raise exception.PortBindingFailed(port_id=port['id']) [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] nova.exception.PortBindingFailed: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. [ 1820.683942] env[62506]: ERROR nova.compute.manager [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] [ 1820.685410] env[62506]: DEBUG nova.compute.utils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1820.685410] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 13.568s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1820.685661] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1820.685737] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1820.685992] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.443s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1820.689504] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Build of instance 4bac9b51-a7cb-4499-be0d-e404039df5f0 was re-scheduled: Binding failed for port 2c1994e5-967d-42ea-932c-c787807208fe, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1820.689970] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1820.690296] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1820.690388] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquired lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1820.690551] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1820.692556] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f17e005-608a-4022-8cdb-307d80c6a13a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.704613] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-258f8017-1142-4fed-bef3-e2a33cabd316 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.722262] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec9e8a15-fc4a-4299-ab02-e59f52e64936 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.734216] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3783322-8444-490c-9bbc-88799dcc806c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.770666] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181213MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1820.770932] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1820.775744] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1820.808513] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1820.808513] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1820.808513] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1820.808697] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1820.808697] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1820.808697] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1820.809864] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1820.809864] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1820.809864] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1820.809864] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1820.809864] env[62506]: DEBUG nova.virt.hardware [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1820.811384] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19c2f5f4-857b-47e5-9267-e5fb49269b5c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.827054] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc6e7898-b2d2-4ea0-9698-6b17258801da {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.921128] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Releasing lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1820.921623] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1820.921821] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1820.922134] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c4939c11-a597-4f37-9e1e-a8098a840cf6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.931560] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ec7f997-641e-45b5-8026-c1518e3ca4af {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1820.961605] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 49423ca9-a31f-44d0-b05e-ab78b6b31321 could not be found. [ 1820.961742] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1820.961957] env[62506]: INFO nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1820.962211] env[62506]: DEBUG oslo.service.loopingcall [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1820.962659] env[62506]: DEBUG nova.compute.manager [-] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1820.962659] env[62506]: DEBUG nova.network.neutron [-] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1821.009506] env[62506]: DEBUG nova.network.neutron [-] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1821.085451] env[62506]: DEBUG nova.compute.manager [req-3e9447f6-df5d-4cd8-bb54-6760a37e3df6 req-ce7bdb9b-c2de-4215-b093-b0b926dc8b0f service nova] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Received event network-vif-deleted-3698a633-4ba4-43b5-8aef-9c4df07c9722 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1821.245928] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1821.345426] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Successfully created port: 1e2672bc-6822-4ac5-a664-248ad0cd6bb6 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1821.459463] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1821.513759] env[62506]: DEBUG nova.network.neutron [-] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1821.525923] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad3c5d8d-d172-4b6a-8d50-1bf9be185501 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.534318] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40bb1d05-774a-4ba6-ab98-c659969cc9cf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.571466] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a8d27d3-e02c-4d1a-93d5-9d3dc8dce6db {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.583554] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30904adb-93b0-4308-b9c0-1510a21dc6b3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.600932] env[62506]: DEBUG nova.compute.provider_tree [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1821.963745] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Releasing lock "refresh_cache-4bac9b51-a7cb-4499-be0d-e404039df5f0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1821.964023] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1821.964206] env[62506]: DEBUG nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1821.964375] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1822.010766] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1822.018713] env[62506]: INFO nova.compute.manager [-] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Took 1.06 seconds to deallocate network for instance. [ 1822.020368] env[62506]: DEBUG nova.compute.claims [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1822.020544] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.104135] env[62506]: DEBUG nova.scheduler.client.report [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1822.514418] env[62506]: DEBUG nova.network.neutron [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1822.610645] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.924s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1822.612468] env[62506]: ERROR nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Traceback (most recent call last): [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self.driver.spawn(context, instance, image_meta, [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] vm_ref = self.build_virtual_machine(instance, [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] vif_infos = vmwarevif.get_vif_info(self._session, [ 1822.612468] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] for vif in network_info: [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return self._sync_wrapper(fn, *args, **kwargs) [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self.wait() [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self[:] = self._gt.wait() [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return self._exit_event.wait() [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] result = hub.switch() [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1822.613014] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return self.greenlet.switch() [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] result = function(*args, **kwargs) [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] return func(*args, **kwargs) [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] raise e [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] nwinfo = self.network_api.allocate_for_instance( [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] created_port_ids = self._update_ports_for_instance( [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] with excutils.save_and_reraise_exception(): [ 1822.613410] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] self.force_reraise() [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] raise self.value [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] updated_port = self._update_port( [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] _ensure_no_port_binding_failure(port) [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] raise exception.PortBindingFailed(port_id=port['id']) [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] nova.exception.PortBindingFailed: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. [ 1822.613936] env[62506]: ERROR nova.compute.manager [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] [ 1822.614278] env[62506]: DEBUG nova.compute.utils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1822.614278] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 12.685s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1822.614396] env[62506]: DEBUG nova.objects.instance [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lazy-loading 'resources' on Instance uuid 3aca933d-f1e7-4dbe-a487-f8954da24981 {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 1822.615625] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Build of instance b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa was re-scheduled: Binding failed for port d2a9bf2f-b6fd-47f4-a2e0-d074ccad54a3, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1822.616099] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1822.616643] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquiring lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1822.616643] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Acquired lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1822.616643] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1822.698298] env[62506]: ERROR nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1822.698298] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1822.698298] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1822.698298] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1822.698298] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1822.698298] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1822.698298] env[62506]: ERROR nova.compute.manager raise self.value [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1822.698298] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1822.698298] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1822.698298] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1822.699058] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1822.699058] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1822.699058] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1822.699058] env[62506]: ERROR nova.compute.manager [ 1822.699058] env[62506]: Traceback (most recent call last): [ 1822.699058] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1822.699058] env[62506]: listener.cb(fileno) [ 1822.699058] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1822.699058] env[62506]: result = function(*args, **kwargs) [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1822.699058] env[62506]: return func(*args, **kwargs) [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1822.699058] env[62506]: raise e [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1822.699058] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1822.699058] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1822.699058] env[62506]: with excutils.save_and_reraise_exception(): [ 1822.699058] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1822.699058] env[62506]: self.force_reraise() [ 1822.699058] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1822.699058] env[62506]: raise self.value [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1822.699058] env[62506]: updated_port = self._update_port( [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1822.699058] env[62506]: _ensure_no_port_binding_failure(port) [ 1822.699058] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1822.699058] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1822.700775] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1822.700775] env[62506]: Removing descriptor: 19 [ 1822.700775] env[62506]: ERROR nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Traceback (most recent call last): [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] yield resources [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self.driver.spawn(context, instance, image_meta, [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1822.700775] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] vm_ref = self.build_virtual_machine(instance, [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] vif_infos = vmwarevif.get_vif_info(self._session, [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] for vif in network_info: [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return self._sync_wrapper(fn, *args, **kwargs) [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self.wait() [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self[:] = self._gt.wait() [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return self._exit_event.wait() [ 1822.701228] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] result = hub.switch() [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return self.greenlet.switch() [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] result = function(*args, **kwargs) [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return func(*args, **kwargs) [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] raise e [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] nwinfo = self.network_api.allocate_for_instance( [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1822.701640] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] created_port_ids = self._update_ports_for_instance( [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] with excutils.save_and_reraise_exception(): [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self.force_reraise() [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] raise self.value [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] updated_port = self._update_port( [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] _ensure_no_port_binding_failure(port) [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1822.702342] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] raise exception.PortBindingFailed(port_id=port['id']) [ 1822.703059] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1822.703059] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] [ 1822.703059] env[62506]: INFO nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Terminating instance [ 1823.018174] env[62506]: INFO nova.compute.manager [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: 4bac9b51-a7cb-4499-be0d-e404039df5f0] Took 1.05 seconds to deallocate network for instance. [ 1823.170837] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1823.205717] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquiring lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1823.207043] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquired lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1823.207043] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1823.373103] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1823.488151] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d36fd3b-f1f2-4bd4-bbb5-f41fbeb1f06c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.495486] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdfedc4f-bc0d-4916-9b7d-1f0653f42490 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.539428] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-430b8362-74c1-41c3-aac3-4e816f5d373a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.548330] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae8358a2-04a2-4407-a8eb-479c88fa6839 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.555467] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7031ddb5-fa9c-44c4-b283-dd8e7fd85301 tempest-ServersListShow296Test-547794886 tempest-ServersListShow296Test-547794886-project-member] Acquiring lock "c168f33c-ab2a-4d6c-900d-9831d0c034fa" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1823.555467] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7031ddb5-fa9c-44c4-b283-dd8e7fd85301 tempest-ServersListShow296Test-547794886 tempest-ServersListShow296Test-547794886-project-member] Lock "c168f33c-ab2a-4d6c-900d-9831d0c034fa" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1823.570358] env[62506]: DEBUG nova.compute.provider_tree [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1823.750263] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1823.841355] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1823.881020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Releasing lock "refresh_cache-b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1823.881020] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1823.881020] env[62506]: DEBUG nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1823.881020] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1823.903470] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1823.911351] env[62506]: DEBUG nova.compute.manager [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Received event network-changed-43739195-3ff0-497a-871e-7dd5f4b9a3f9 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1823.911548] env[62506]: DEBUG nova.compute.manager [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Refreshing instance network info cache due to event network-changed-43739195-3ff0-497a-871e-7dd5f4b9a3f9. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1823.911735] env[62506]: DEBUG oslo_concurrency.lockutils [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] Acquiring lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1824.074550] env[62506]: ERROR nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1824.074550] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1824.074550] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1824.074550] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1824.074550] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1824.074550] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1824.074550] env[62506]: ERROR nova.compute.manager raise self.value [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1824.074550] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1824.074550] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1824.074550] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1824.075603] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1824.075603] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1824.075603] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1824.075603] env[62506]: ERROR nova.compute.manager [ 1824.075603] env[62506]: Traceback (most recent call last): [ 1824.075603] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1824.075603] env[62506]: listener.cb(fileno) [ 1824.075603] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1824.075603] env[62506]: result = function(*args, **kwargs) [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1824.075603] env[62506]: return func(*args, **kwargs) [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1824.075603] env[62506]: raise e [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1824.075603] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1824.075603] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1824.075603] env[62506]: with excutils.save_and_reraise_exception(): [ 1824.075603] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1824.075603] env[62506]: self.force_reraise() [ 1824.075603] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1824.075603] env[62506]: raise self.value [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1824.075603] env[62506]: updated_port = self._update_port( [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1824.075603] env[62506]: _ensure_no_port_binding_failure(port) [ 1824.075603] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1824.075603] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1824.076436] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1824.076436] env[62506]: Removing descriptor: 15 [ 1824.076436] env[62506]: DEBUG nova.scheduler.client.report [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1824.077651] env[62506]: ERROR nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Traceback (most recent call last): [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] yield resources [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self.driver.spawn(context, instance, image_meta, [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] vm_ref = self.build_virtual_machine(instance, [ 1824.077651] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] vif_infos = vmwarevif.get_vif_info(self._session, [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] for vif in network_info: [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return self._sync_wrapper(fn, *args, **kwargs) [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self.wait() [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self[:] = self._gt.wait() [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return self._exit_event.wait() [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1824.078056] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] result = hub.switch() [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return self.greenlet.switch() [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] result = function(*args, **kwargs) [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return func(*args, **kwargs) [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] raise e [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] nwinfo = self.network_api.allocate_for_instance( [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] created_port_ids = self._update_ports_for_instance( [ 1824.078616] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] with excutils.save_and_reraise_exception(): [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self.force_reraise() [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] raise self.value [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] updated_port = self._update_port( [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] _ensure_no_port_binding_failure(port) [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] raise exception.PortBindingFailed(port_id=port['id']) [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1824.081393] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] [ 1824.081841] env[62506]: INFO nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Terminating instance [ 1824.100254] env[62506]: INFO nova.scheduler.client.report [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Deleted allocations for instance 4bac9b51-a7cb-4499-be0d-e404039df5f0 [ 1824.349343] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Releasing lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1824.349827] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1824.350036] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1824.350418] env[62506]: DEBUG oslo_concurrency.lockutils [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] Acquired lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1824.350608] env[62506]: DEBUG nova.network.neutron [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Refreshing network info cache for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1824.351681] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-48f4c412-878c-4335-af8c-5de5c29da5d2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1824.362477] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-365bbb55-545b-43e8-9b4b-fca683f4517c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1824.386082] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0 could not be found. [ 1824.386511] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1824.386511] env[62506]: INFO nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1824.386681] env[62506]: DEBUG oslo.service.loopingcall [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1824.386883] env[62506]: DEBUG nova.compute.manager [-] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1824.386883] env[62506]: DEBUG nova.network.neutron [-] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1824.410184] env[62506]: DEBUG nova.network.neutron [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1824.418713] env[62506]: DEBUG nova.network.neutron [-] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1824.583283] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.969s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1824.586555] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.757s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1824.590265] env[62506]: INFO nova.compute.claims [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1824.602018] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquiring lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1824.602018] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquired lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1824.602018] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1824.613057] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aad61f56-438f-4eda-ab9b-5c898ffca737 tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "4bac9b51-a7cb-4499-be0d-e404039df5f0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 35.484s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1824.620563] env[62506]: INFO nova.scheduler.client.report [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Deleted allocations for instance 3aca933d-f1e7-4dbe-a487-f8954da24981 [ 1824.889896] env[62506]: DEBUG nova.network.neutron [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1824.915201] env[62506]: INFO nova.compute.manager [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] [instance: b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa] Took 1.03 seconds to deallocate network for instance. [ 1824.921527] env[62506]: DEBUG nova.network.neutron [-] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1825.030343] env[62506]: DEBUG nova.network.neutron [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1825.125596] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1825.134973] env[62506]: DEBUG oslo_concurrency.lockutils [None req-bb25da0a-9116-4ee9-a634-284195dda483 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "3aca933d-f1e7-4dbe-a487-f8954da24981" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 19.718s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1825.143610] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1825.247369] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1825.426117] env[62506]: INFO nova.compute.manager [-] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Took 1.04 seconds to deallocate network for instance. [ 1825.428228] env[62506]: DEBUG nova.compute.claims [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1825.428451] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1825.533293] env[62506]: DEBUG oslo_concurrency.lockutils [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] Releasing lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1825.533487] env[62506]: DEBUG nova.compute.manager [req-2b57d7c8-4489-417f-bf9c-e2b8369102ab req-0e2f83fa-a0a4-4d25-89f0-df22cdcd6f38 service nova] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Received event network-vif-deleted-43739195-3ff0-497a-871e-7dd5f4b9a3f9 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1825.666522] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1825.752262] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Releasing lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1825.752697] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1825.753272] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1825.753272] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b329c03a-07c1-46d7-82e5-bb90ce2d9847 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1825.770835] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b880bb9-f4cc-4d47-80ef-2a503d0f47f5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1825.809363] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance cb3b9b8c-4a27-430d-8653-f1afe3ac52cd could not be found. [ 1825.809363] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1825.809363] env[62506]: INFO nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1825.809363] env[62506]: DEBUG oslo.service.loopingcall [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1825.809363] env[62506]: DEBUG nova.compute.manager [-] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1825.809363] env[62506]: DEBUG nova.network.neutron [-] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1825.858776] env[62506]: DEBUG nova.network.neutron [-] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1825.886531] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquiring lock "e8433dc4-7c83-4288-acf4-5f5a2863592c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1825.886531] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Lock "e8433dc4-7c83-4288-acf4-5f5a2863592c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1825.975128] env[62506]: INFO nova.scheduler.client.report [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Deleted allocations for instance b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa [ 1826.043714] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b48feec-fc79-4fa4-a6b7-708f2c6bdd59 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1826.061437] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e343b3f7-b685-42b9-b2d6-c131df20877f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1826.101427] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fddfc6d-c0f9-4489-abb2-1663e78e448b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1826.109948] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcf7945c-7fd5-4442-994d-7819485d82ff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1826.122953] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1826.364866] env[62506]: DEBUG nova.network.neutron [-] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1826.488301] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6379ad8d-523d-4dd3-a647-5349574c53fb tempest-ServersAdminTestJSON-748031710 tempest-ServersAdminTestJSON-748031710-project-member] Lock "b17e7d01-93e2-4b1a-acd0-a344c5b4b1aa" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 34.645s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1826.559242] env[62506]: DEBUG nova.compute.manager [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Received event network-changed-1e2672bc-6822-4ac5-a664-248ad0cd6bb6 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1826.559242] env[62506]: DEBUG nova.compute.manager [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Refreshing instance network info cache due to event network-changed-1e2672bc-6822-4ac5-a664-248ad0cd6bb6. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1826.559242] env[62506]: DEBUG oslo_concurrency.lockutils [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] Acquiring lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1826.559242] env[62506]: DEBUG oslo_concurrency.lockutils [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] Acquired lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1826.559242] env[62506]: DEBUG nova.network.neutron [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Refreshing network info cache for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1826.662207] env[62506]: ERROR nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [req-2186fd96-94db-4aa2-8273-e066005313d5] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-2186fd96-94db-4aa2-8273-e066005313d5"}]} [ 1826.686814] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1826.708879] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1826.709188] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1826.728264] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1826.761292] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1826.873267] env[62506]: INFO nova.compute.manager [-] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Took 1.06 seconds to deallocate network for instance. [ 1826.875766] env[62506]: DEBUG nova.compute.claims [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1826.875980] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1826.992988] env[62506]: DEBUG nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1827.099344] env[62506]: DEBUG nova.network.neutron [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1827.128782] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-badc88ed-24c8-4fe1-b6c5-50a0b69e9729 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1827.145861] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d742f47-5649-4fdc-b53d-422e7e872f53 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1827.192284] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2485982a-2bc1-41cf-8c6f-e1f0e1b16131 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1827.201604] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae7845d4-e83d-436f-a963-d0baf584c27d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1827.216698] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1827.323160] env[62506]: DEBUG nova.network.neutron [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1827.534020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1827.758529] env[62506]: ERROR nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [req-84ba8abd-65c9-4091-9a18-d137f42ea45d] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-84ba8abd-65c9-4091-9a18-d137f42ea45d"}]} [ 1827.789308] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1827.820315] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1827.820548] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1827.826500] env[62506]: DEBUG oslo_concurrency.lockutils [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] Releasing lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1827.826500] env[62506]: DEBUG nova.compute.manager [req-b4ce662e-22f3-46ab-a7fa-e958c807d127 req-78b0fcc9-9604-4b1e-bdc3-1715107bc125 service nova] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Received event network-vif-deleted-1e2672bc-6822-4ac5-a664-248ad0cd6bb6 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1827.850060] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1827.881646] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1828.107053] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquiring lock "1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1828.107053] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Lock "1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1828.226096] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90ebaa40-12a7-4da2-9a19-96b0655b6e98 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1828.237379] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85850265-3ef3-4c91-b981-f7ab47580246 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1828.269815] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-909a0a84-6f47-425b-9bf3-a29732951544 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1828.278554] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acb62137-ef6e-49f9-ae0f-67f43b1e3fbe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1828.293574] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1828.845215] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 33 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1828.845215] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 33 to 34 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1828.845622] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1829.353653] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 4.767s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1829.353744] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1829.359752] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.404s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1829.864975] env[62506]: DEBUG nova.compute.utils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1829.874895] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1829.874895] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1830.147772] env[62506]: DEBUG nova.policy [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4b57f65825f149d3a9582dbe265fa270', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9ac5d97da69447b7a0be02942668d1a2', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1830.205181] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-287d546e-786c-4937-87fc-71e92d2f332f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1830.214405] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92c4c7ca-33cf-45b8-91d6-30ffd08cc9d0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1830.250987] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-974ed021-ed4f-44a4-b611-f63c44498a84 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1830.260140] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dcaf833-0763-4024-ad1c-e21a45f58a9f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1830.275871] env[62506]: DEBUG nova.compute.provider_tree [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1830.377499] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1830.627822] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Successfully created port: a0aeea44-cf68-4788-ae5d-b1eef63e32c2 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1830.779429] env[62506]: DEBUG nova.scheduler.client.report [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1831.284570] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.927s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1831.286658] env[62506]: ERROR nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Traceback (most recent call last): [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self.driver.spawn(context, instance, image_meta, [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] vm_ref = self.build_virtual_machine(instance, [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1831.286658] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] for vif in network_info: [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return self._sync_wrapper(fn, *args, **kwargs) [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self.wait() [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self[:] = self._gt.wait() [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return self._exit_event.wait() [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] result = hub.switch() [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1831.287055] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return self.greenlet.switch() [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] result = function(*args, **kwargs) [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] return func(*args, **kwargs) [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] raise e [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] nwinfo = self.network_api.allocate_for_instance( [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] created_port_ids = self._update_ports_for_instance( [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] with excutils.save_and_reraise_exception(): [ 1831.290100] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] self.force_reraise() [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] raise self.value [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] updated_port = self._update_port( [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] _ensure_no_port_binding_failure(port) [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] raise exception.PortBindingFailed(port_id=port['id']) [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] nova.exception.PortBindingFailed: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. [ 1831.290557] env[62506]: ERROR nova.compute.manager [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] [ 1831.290961] env[62506]: DEBUG nova.compute.utils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1831.294141] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.215s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1831.294929] env[62506]: INFO nova.compute.claims [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1831.297064] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Build of instance 54e34c6f-3863-4683-b301-a6a9d25f1c4d was re-scheduled: Binding failed for port 8c510593-5dc7-4c66-860a-98fed06ca332, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1831.297556] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1831.298212] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquiring lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1831.298212] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Acquired lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1831.298212] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1831.390862] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1831.423979] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:59:40Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='1561477604',id=25,is_public=True,memory_mb=192,name='tempest-flavor_with_ephemeral_0-1787697303',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1831.424620] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1831.424620] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1831.424910] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1831.424951] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1831.425549] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1831.425549] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1831.425732] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1831.425732] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1831.425996] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1831.426100] env[62506]: DEBUG nova.virt.hardware [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1831.429342] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4eeadc25-e392-4bd4-862a-27238a951a3f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.437882] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74ea5b12-61c3-4a96-9f3c-6f18f2202ed5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.859105] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1832.076138] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1832.581934] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Releasing lock "refresh_cache-54e34c6f-3863-4683-b301-a6a9d25f1c4d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1832.583140] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1832.583440] env[62506]: DEBUG nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1832.583645] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1832.631267] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-530fc4ec-2b97-4bf8-8acb-206e9ee5d2b4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.640280] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d24bcb2-d75b-4472-a636-f627500ddb6b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.684365] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1832.685297] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d544f80-5c44-4051-bb5d-ee0128c7f43c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.693978] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80ee143c-7601-4d4d-b412-31fbcb3c2afa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.709591] env[62506]: DEBUG nova.compute.provider_tree [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1833.089162] env[62506]: ERROR nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1833.089162] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1833.089162] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1833.089162] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1833.089162] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1833.089162] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1833.089162] env[62506]: ERROR nova.compute.manager raise self.value [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1833.089162] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1833.089162] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1833.089162] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1833.089731] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1833.089731] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1833.089731] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1833.089731] env[62506]: ERROR nova.compute.manager [ 1833.089731] env[62506]: Traceback (most recent call last): [ 1833.089731] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1833.089731] env[62506]: listener.cb(fileno) [ 1833.089731] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1833.089731] env[62506]: result = function(*args, **kwargs) [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1833.089731] env[62506]: return func(*args, **kwargs) [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1833.089731] env[62506]: raise e [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1833.089731] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1833.089731] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1833.089731] env[62506]: with excutils.save_and_reraise_exception(): [ 1833.089731] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1833.089731] env[62506]: self.force_reraise() [ 1833.089731] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1833.089731] env[62506]: raise self.value [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1833.089731] env[62506]: updated_port = self._update_port( [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1833.089731] env[62506]: _ensure_no_port_binding_failure(port) [ 1833.089731] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1833.089731] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1833.090670] env[62506]: nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1833.090670] env[62506]: Removing descriptor: 15 [ 1833.090670] env[62506]: ERROR nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Traceback (most recent call last): [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] yield resources [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self.driver.spawn(context, instance, image_meta, [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1833.090670] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] vm_ref = self.build_virtual_machine(instance, [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] for vif in network_info: [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return self._sync_wrapper(fn, *args, **kwargs) [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self.wait() [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self[:] = self._gt.wait() [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return self._exit_event.wait() [ 1833.091134] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] result = hub.switch() [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return self.greenlet.switch() [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] result = function(*args, **kwargs) [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return func(*args, **kwargs) [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] raise e [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] nwinfo = self.network_api.allocate_for_instance( [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1833.091540] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] created_port_ids = self._update_ports_for_instance( [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] with excutils.save_and_reraise_exception(): [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self.force_reraise() [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] raise self.value [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] updated_port = self._update_port( [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] _ensure_no_port_binding_failure(port) [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1833.091918] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] raise exception.PortBindingFailed(port_id=port['id']) [ 1833.092321] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1833.092321] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] [ 1833.092321] env[62506]: INFO nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Terminating instance [ 1833.189260] env[62506]: DEBUG nova.network.neutron [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1833.214493] env[62506]: DEBUG nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1833.594621] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquiring lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1833.595044] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquired lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1833.595044] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1833.693237] env[62506]: INFO nova.compute.manager [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] [instance: 54e34c6f-3863-4683-b301-a6a9d25f1c4d] Took 1.11 seconds to deallocate network for instance. [ 1833.720198] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.428s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1833.720819] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1833.725051] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.383s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1833.731115] env[62506]: INFO nova.compute.claims [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1834.121448] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1834.209213] env[62506]: DEBUG nova.compute.manager [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Received event network-changed-a0aeea44-cf68-4788-ae5d-b1eef63e32c2 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1834.209213] env[62506]: DEBUG nova.compute.manager [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Refreshing instance network info cache due to event network-changed-a0aeea44-cf68-4788-ae5d-b1eef63e32c2. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1834.209373] env[62506]: DEBUG oslo_concurrency.lockutils [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] Acquiring lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1834.239967] env[62506]: DEBUG nova.compute.utils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1834.242023] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1834.245363] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1834.245487] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1834.301699] env[62506]: DEBUG nova.policy [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ab67b57f49ad47ea9aab58b4f7bc563f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c97e56471ed471aa4a0812ec92e082c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1834.464515] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "7e185b7e-8a95-4931-96a8-1c27516f767d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1834.464868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "7e185b7e-8a95-4931-96a8-1c27516f767d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1834.737624] env[62506]: INFO nova.scheduler.client.report [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Deleted allocations for instance 54e34c6f-3863-4683-b301-a6a9d25f1c4d [ 1834.757802] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1834.761997] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Releasing lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1834.762679] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1834.763329] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1834.763476] env[62506]: DEBUG oslo_concurrency.lockutils [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] Acquired lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1834.763668] env[62506]: DEBUG nova.network.neutron [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Refreshing network info cache for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1834.764669] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9cd8eb9f-9945-4b22-a15c-2ab6035ddfd6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1834.780205] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bb2684e-0639-4456-8690-ab0a1635da7a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1834.812730] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4a95ce6c-d1c0-4677-a330-70620a05989d could not be found. [ 1834.813073] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1834.813234] env[62506]: INFO nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1834.813506] env[62506]: DEBUG oslo.service.loopingcall [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1834.813963] env[62506]: DEBUG nova.compute.manager [-] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1834.814075] env[62506]: DEBUG nova.network.neutron [-] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1834.847916] env[62506]: DEBUG nova.network.neutron [-] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1835.091017] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e823ba5-3cf8-4e2d-9b68-dc9fd49719b9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.098294] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f003dbd3-d9d1-435c-907b-43708091b643 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.135422] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e286748-8589-44b2-a24b-9b26f270dd01 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.144223] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92f80962-e7f6-477e-b578-0376fc426ae6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.164381] env[62506]: DEBUG nova.compute.provider_tree [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1835.210070] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Successfully created port: 812e694a-b615-422a-bfb7-829c81f957ff {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1835.246294] env[62506]: DEBUG oslo_concurrency.lockutils [None req-44adfcbe-cf1e-4cde-96f2-2ddf07cf4f17 tempest-VolumesAssistedSnapshotsTest-825925935 tempest-VolumesAssistedSnapshotsTest-825925935-project-member] Lock "54e34c6f-3863-4683-b301-a6a9d25f1c4d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 40.330s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1835.307878] env[62506]: DEBUG nova.network.neutron [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1835.350205] env[62506]: DEBUG nova.network.neutron [-] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1835.667890] env[62506]: DEBUG nova.scheduler.client.report [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1835.730868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquiring lock "e0de75e0-ca8e-4525-ae7f-ed13fcd07338" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1835.731140] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Lock "e0de75e0-ca8e-4525-ae7f-ed13fcd07338" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1835.749484] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1835.771917] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1835.812806] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1835.813158] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1835.813430] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1835.813701] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1835.813927] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1835.814162] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1835.814444] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1835.814658] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1835.814901] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1835.815120] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1835.816144] env[62506]: DEBUG nova.virt.hardware [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1835.817377] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0277834a-d097-434a-8399-6fb6bd4158f0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.826048] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2a4053c-3d48-449c-a501-0de486c55c75 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.852249] env[62506]: INFO nova.compute.manager [-] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Took 1.04 seconds to deallocate network for instance. [ 1835.854478] env[62506]: DEBUG nova.compute.claims [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1835.854728] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1835.907559] env[62506]: DEBUG nova.network.neutron [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1836.175508] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.450s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1836.176108] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1836.184030] env[62506]: DEBUG oslo_concurrency.lockutils [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 20.633s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1836.184240] env[62506]: DEBUG oslo_concurrency.lockutils [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1836.184294] env[62506]: INFO nova.compute.manager [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Successfully reverted task state from image_uploading on failure for instance. [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server [None req-68180b8f-f37e-4c4f-bc58-2e16169113f0 tempest-ServersAaction247Test-477488001 tempest-ServersAaction247Test-477488001-project-member] Exception during message handling: oslo_vmware.exceptions.ManagedObjectNotFoundException: The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created [ 1836.192936] env[62506]: Cause: Server raised fault: 'The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created' [ 1836.192936] env[62506]: Faults: [ManagedObjectNotFound] [ 1836.192936] env[62506]: Details: {'obj': 'snapshot-446957'} [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py", line 377, in request_handler [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server response = request(managed_object, **kwargs) [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/suds/client.py", line 586, in __call__ [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server return client.invoke(args, kwargs) [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/suds/client.py", line 728, in invoke [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server result = self.send(soapenv, timeout=timeout) [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/suds/client.py", line 777, in send [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server return self.process_reply(reply.message, None, None) [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/suds/client.py", line 840, in process_reply [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server raise WebFault(fault, replyroot) [ 1836.192936] env[62506]: ERROR oslo_messaging.rpc.server suds.WebFault: Server raised fault: 'The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created' [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 301, in _invoke_api [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server return api_method(*args, **kwargs) [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py", line 413, in request_handler [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server raise exceptions.VimFaultException(fault_list, fault_string, [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server oslo_vmware.exceptions.VimFaultException: The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server Cause: Server raised fault: 'The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created' [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server Faults: [ManagedObjectNotFound] [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server Details: {'obj': 'snapshot-446957'} [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 174, in _process_incoming [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1836.193474] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server raise self.value [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 168, in decorated_function [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server raise self.value [ 1836.194049] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 159, in decorated_function [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1484, in decorated_function [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 215, in decorated_function [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server raise self.value [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 205, in decorated_function [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 4507, in backup_instance [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server self._do_snapshot_instance(context, image_id, instance) [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 235, in decorated_function [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1836.194589] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server raise self.value [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 232, in decorated_function [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server return function(self, context, image_id, instance, [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 4493, in _do_snapshot_instance [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server self._snapshot_instance(context, image_id, instance, [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 4576, in _snapshot_instance [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server self.driver.snapshot(context, instance, image_id, [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 573, in snapshot [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server self._vmops.snapshot(context, instance, image_id, update_task_state) [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 1033, in snapshot [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server self._delete_vm_snapshot(instance, vm_ref, snapshot_ref) [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/decorator.py", line 232, in fun [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server return caller(func, *(extras + args), **kw) [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 124, in retry_if_task_in_progress [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server f(*args, **kwargs) [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 933, in _delete_vm_snapshot [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server delete_snapshot_task = self._session._call_method( [ 1836.195107] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 127, in _call_method [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception() as ctxt: [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server raise self.value [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 125, in _call_method [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server return self.invoke_api(module, method, *args, **kwargs) [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 358, in invoke_api [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server return _invoke_api(module, method, *args, **kwargs) [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 122, in func [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 122, in _inner [ 1836.195654] env[62506]: ERROR oslo_messaging.rpc.server idle = self.f(*self.args, **self.kw) [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 96, in _func [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 341, in _invoke_api [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server raise clazz(str(excep), [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server oslo_vmware.exceptions.ManagedObjectNotFoundException: The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server Cause: Server raised fault: 'The object 'vim.vm.Snapshot:snapshot-446957' has already been deleted or has not been completely created' [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server Faults: [ManagedObjectNotFound] [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server Details: {'obj': 'snapshot-446957'} [ 1836.197197] env[62506]: ERROR oslo_messaging.rpc.server [ 1836.197197] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.929s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1836.290305] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1836.409811] env[62506]: DEBUG oslo_concurrency.lockutils [req-80731a80-4fd1-4edb-8fbb-706aad23692f req-65d78d9e-4f69-49c1-a757-dd31198b3881 service nova] Releasing lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1836.705656] env[62506]: DEBUG nova.compute.utils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1836.711871] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1836.711871] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1836.941465] env[62506]: DEBUG nova.policy [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e10051d6e2714066b426d529020e6e5e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9e07b229f35c4dfa834ef283d60cb26d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1836.997391] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90541029-4ce3-4e75-87a0-d76412cf237a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.006148] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f1ee5c5-2ab8-4ce9-a222-7add11a1ce74 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.039277] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-701395cc-b16c-4dd1-8907-d6ed7718b43f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.047102] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1563b6fc-9fea-4c8c-a4e5-231e31960492 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1837.060528] env[62506]: DEBUG nova.compute.provider_tree [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1837.209963] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1837.565610] env[62506]: DEBUG nova.scheduler.client.report [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1838.072625] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.878s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1838.073392] env[62506]: ERROR nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Traceback (most recent call last): [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self.driver.spawn(context, instance, image_meta, [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] vm_ref = self.build_virtual_machine(instance, [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1838.073392] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] for vif in network_info: [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return self._sync_wrapper(fn, *args, **kwargs) [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self.wait() [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self[:] = self._gt.wait() [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return self._exit_event.wait() [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] result = hub.switch() [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1838.073902] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return self.greenlet.switch() [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] result = function(*args, **kwargs) [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] return func(*args, **kwargs) [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] raise e [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] nwinfo = self.network_api.allocate_for_instance( [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] created_port_ids = self._update_ports_for_instance( [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] with excutils.save_and_reraise_exception(): [ 1838.074678] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] self.force_reraise() [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] raise self.value [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] updated_port = self._update_port( [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] _ensure_no_port_binding_failure(port) [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] raise exception.PortBindingFailed(port_id=port['id']) [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] nova.exception.PortBindingFailed: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. [ 1838.075076] env[62506]: ERROR nova.compute.manager [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] [ 1838.078469] env[62506]: DEBUG nova.compute.utils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1838.079796] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 17.309s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1838.082024] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Build of instance c16e39bd-3443-4dbb-8d5a-ab90cf27251f was re-scheduled: Binding failed for port 3cf9e11d-bf4f-4884-85f2-3e34051ae260, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1838.082024] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1838.082268] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquiring lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1838.082417] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Acquired lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1838.082577] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1838.220748] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1838.261857] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1838.261857] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1838.262121] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1838.262162] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1838.262320] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1838.262500] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1838.262850] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1838.262850] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1838.262956] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1838.263819] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1838.264056] env[62506]: DEBUG nova.virt.hardware [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1838.265113] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0268e81-1d60-4ad8-9b01-f60fea162683 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1838.275038] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0007cec1-8e23-499a-8aa9-bebddb5c4d03 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1838.404250] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "03e1ebe5-7a02-430e-b38c-d215e85d59ae" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1838.404512] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "03e1ebe5-7a02-430e-b38c-d215e85d59ae" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1838.550214] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Successfully created port: 4acc1eca-d65e-4e91-a872-24d025379936 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1838.621017] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1838.730215] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1838.883249] env[62506]: DEBUG nova.compute.manager [req-cfe37066-01e1-4cec-843a-04031eacbddf req-5a7f48be-b12f-4a47-a271-d2b9258e713f service nova] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Received event network-vif-deleted-a0aeea44-cf68-4788-ae5d-b1eef63e32c2 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1839.230862] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Releasing lock "refresh_cache-c16e39bd-3443-4dbb-8d5a-ab90cf27251f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1839.231480] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1839.231581] env[62506]: DEBUG nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1839.231855] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1839.256506] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1839.640821] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance c16e39bd-3443-4dbb-8d5a-ab90cf27251f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1839.641455] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 49423ca9-a31f-44d0-b05e-ab78b6b31321 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1839.641654] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1839.641816] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance cb3b9b8c-4a27-430d-8653-f1afe3ac52cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1839.641964] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 4a95ce6c-d1c0-4677-a330-70620a05989d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1839.642125] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b5527c82-048e-4738-8878-0807aa609596 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1839.642270] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e433b5a0-2742-4897-a7d9-46571bcc0f2c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1839.691924] env[62506]: ERROR nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1839.691924] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1839.691924] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1839.691924] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1839.691924] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1839.691924] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1839.691924] env[62506]: ERROR nova.compute.manager raise self.value [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1839.691924] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1839.691924] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1839.691924] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1839.692510] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1839.692510] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1839.692510] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1839.692510] env[62506]: ERROR nova.compute.manager [ 1839.692510] env[62506]: Traceback (most recent call last): [ 1839.692510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1839.692510] env[62506]: listener.cb(fileno) [ 1839.692510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1839.692510] env[62506]: result = function(*args, **kwargs) [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1839.692510] env[62506]: return func(*args, **kwargs) [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1839.692510] env[62506]: raise e [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1839.692510] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1839.692510] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1839.692510] env[62506]: with excutils.save_and_reraise_exception(): [ 1839.692510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1839.692510] env[62506]: self.force_reraise() [ 1839.692510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1839.692510] env[62506]: raise self.value [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1839.692510] env[62506]: updated_port = self._update_port( [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1839.692510] env[62506]: _ensure_no_port_binding_failure(port) [ 1839.692510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1839.692510] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1839.693456] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1839.693456] env[62506]: Removing descriptor: 15 [ 1839.694757] env[62506]: ERROR nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] Traceback (most recent call last): [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] yield resources [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self.driver.spawn(context, instance, image_meta, [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] vm_ref = self.build_virtual_machine(instance, [ 1839.694757] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] vif_infos = vmwarevif.get_vif_info(self._session, [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] for vif in network_info: [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return self._sync_wrapper(fn, *args, **kwargs) [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self.wait() [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self[:] = self._gt.wait() [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return self._exit_event.wait() [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1839.695208] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] result = hub.switch() [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return self.greenlet.switch() [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] result = function(*args, **kwargs) [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return func(*args, **kwargs) [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] raise e [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] nwinfo = self.network_api.allocate_for_instance( [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] created_port_ids = self._update_ports_for_instance( [ 1839.695637] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] with excutils.save_and_reraise_exception(): [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self.force_reraise() [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] raise self.value [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] updated_port = self._update_port( [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] _ensure_no_port_binding_failure(port) [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] raise exception.PortBindingFailed(port_id=port['id']) [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1839.696099] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] [ 1839.696566] env[62506]: INFO nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Terminating instance [ 1839.765936] env[62506]: DEBUG nova.network.neutron [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1840.146944] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance af042cab-9132-4425-b017-133ec85afa0c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1840.200028] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1840.200255] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1840.200463] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1840.268038] env[62506]: INFO nova.compute.manager [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] [instance: c16e39bd-3443-4dbb-8d5a-ab90cf27251f] Took 1.04 seconds to deallocate network for instance. [ 1840.653642] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 386935e4-c2b7-4207-9670-cd532ff12f70 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1840.750449] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1840.993509] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1841.157823] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 88322331-7cb9-46a9-8e8c-82f0198c90d8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1841.177250] env[62506]: ERROR nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1841.177250] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1841.177250] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1841.177250] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1841.177250] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1841.177250] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1841.177250] env[62506]: ERROR nova.compute.manager raise self.value [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1841.177250] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1841.177250] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1841.177250] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1841.177608] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1841.177608] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1841.177608] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1841.177608] env[62506]: ERROR nova.compute.manager [ 1841.177608] env[62506]: Traceback (most recent call last): [ 1841.177608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1841.177608] env[62506]: listener.cb(fileno) [ 1841.177608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1841.177608] env[62506]: result = function(*args, **kwargs) [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1841.177608] env[62506]: return func(*args, **kwargs) [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1841.177608] env[62506]: raise e [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1841.177608] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1841.177608] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1841.177608] env[62506]: with excutils.save_and_reraise_exception(): [ 1841.177608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1841.177608] env[62506]: self.force_reraise() [ 1841.177608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1841.177608] env[62506]: raise self.value [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1841.177608] env[62506]: updated_port = self._update_port( [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1841.177608] env[62506]: _ensure_no_port_binding_failure(port) [ 1841.177608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1841.177608] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1841.178307] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1841.178307] env[62506]: Removing descriptor: 19 [ 1841.178307] env[62506]: ERROR nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Traceback (most recent call last): [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] yield resources [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self.driver.spawn(context, instance, image_meta, [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1841.178307] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] vm_ref = self.build_virtual_machine(instance, [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] vif_infos = vmwarevif.get_vif_info(self._session, [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] for vif in network_info: [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return self._sync_wrapper(fn, *args, **kwargs) [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self.wait() [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self[:] = self._gt.wait() [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return self._exit_event.wait() [ 1841.178574] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] result = hub.switch() [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return self.greenlet.switch() [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] result = function(*args, **kwargs) [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return func(*args, **kwargs) [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] raise e [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] nwinfo = self.network_api.allocate_for_instance( [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1841.178922] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] created_port_ids = self._update_ports_for_instance( [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] with excutils.save_and_reraise_exception(): [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self.force_reraise() [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] raise self.value [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] updated_port = self._update_port( [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] _ensure_no_port_binding_failure(port) [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1841.179193] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] raise exception.PortBindingFailed(port_id=port['id']) [ 1841.179463] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1841.179463] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] [ 1841.179463] env[62506]: INFO nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Terminating instance [ 1841.311585] env[62506]: INFO nova.scheduler.client.report [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Deleted allocations for instance c16e39bd-3443-4dbb-8d5a-ab90cf27251f [ 1841.501082] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1841.501189] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1841.501368] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1841.501675] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-80aea6e8-d783-41ce-b143-29c05a655e3c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1841.510801] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-454694e4-80a9-4e5f-bdbc-d9313d12da82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1841.534993] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b5527c82-048e-4738-8878-0807aa609596 could not be found. [ 1841.535247] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1841.535436] env[62506]: INFO nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1841.535685] env[62506]: DEBUG oslo.service.loopingcall [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1841.535913] env[62506]: DEBUG nova.compute.manager [-] [instance: b5527c82-048e-4738-8878-0807aa609596] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1841.536017] env[62506]: DEBUG nova.network.neutron [-] [instance: b5527c82-048e-4738-8878-0807aa609596] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1841.556550] env[62506]: DEBUG nova.network.neutron [-] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1841.662940] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 6cdfeb65-c53f-4ba6-873c-88e593909878 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1841.682888] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquiring lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1841.683114] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquired lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1841.683968] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1841.788619] env[62506]: DEBUG nova.compute.manager [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: b5527c82-048e-4738-8878-0807aa609596] Received event network-changed-812e694a-b615-422a-bfb7-829c81f957ff {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1841.788877] env[62506]: DEBUG nova.compute.manager [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: b5527c82-048e-4738-8878-0807aa609596] Refreshing instance network info cache due to event network-changed-812e694a-b615-422a-bfb7-829c81f957ff. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1841.789165] env[62506]: DEBUG oslo_concurrency.lockutils [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] Acquiring lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1841.789428] env[62506]: DEBUG oslo_concurrency.lockutils [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] Acquired lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1841.789657] env[62506]: DEBUG nova.network.neutron [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: b5527c82-048e-4738-8878-0807aa609596] Refreshing network info cache for port 812e694a-b615-422a-bfb7-829c81f957ff {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1841.819582] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6bb62e59-ea75-4803-b473-efcd38548eae tempest-ServerExternalEventsTest-498447876 tempest-ServerExternalEventsTest-498447876-project-member] Lock "c16e39bd-3443-4dbb-8d5a-ab90cf27251f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 40.608s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1841.827522] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquiring lock "43d09ce7-7946-43d1-8729-d7068c1cdec4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1841.827820] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Lock "43d09ce7-7946-43d1-8729-d7068c1cdec4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1842.059070] env[62506]: DEBUG nova.network.neutron [-] [instance: b5527c82-048e-4738-8878-0807aa609596] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1842.166108] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 249e5265-86b3-4ba8-96b9-d21ea0a5599d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1842.210424] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1842.328149] env[62506]: DEBUG nova.network.neutron [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1842.330257] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1842.479178] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1842.556918] env[62506]: DEBUG nova.network.neutron [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: b5527c82-048e-4738-8878-0807aa609596] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1842.562029] env[62506]: INFO nova.compute.manager [-] [instance: b5527c82-048e-4738-8878-0807aa609596] Took 1.03 seconds to deallocate network for instance. [ 1842.564865] env[62506]: DEBUG nova.compute.claims [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1842.565376] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1842.669839] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b4bcab92-0007-4952-8d74-c470049edd71 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1842.855860] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1842.982247] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Releasing lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1842.982687] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1842.982889] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1842.984065] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8586465d-54a3-4ca3-b347-cc72d3cbfaa5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1842.997324] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b95e4b10-93df-4186-b172-72776ae6bef7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1843.020160] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e433b5a0-2742-4897-a7d9-46571bcc0f2c could not be found. [ 1843.020584] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1843.020816] env[62506]: INFO nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1843.021080] env[62506]: DEBUG oslo.service.loopingcall [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1843.021324] env[62506]: DEBUG nova.compute.manager [-] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1843.021421] env[62506]: DEBUG nova.network.neutron [-] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1843.049822] env[62506]: DEBUG nova.network.neutron [-] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1843.060452] env[62506]: DEBUG oslo_concurrency.lockutils [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] Releasing lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1843.060702] env[62506]: DEBUG nova.compute.manager [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: b5527c82-048e-4738-8878-0807aa609596] Received event network-vif-deleted-812e694a-b615-422a-bfb7-829c81f957ff {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1843.060892] env[62506]: DEBUG nova.compute.manager [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Received event network-changed-4acc1eca-d65e-4e91-a872-24d025379936 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1843.061181] env[62506]: DEBUG nova.compute.manager [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Refreshing instance network info cache due to event network-changed-4acc1eca-d65e-4e91-a872-24d025379936. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1843.061314] env[62506]: DEBUG oslo_concurrency.lockutils [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] Acquiring lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1843.061406] env[62506]: DEBUG oslo_concurrency.lockutils [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] Acquired lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1843.061592] env[62506]: DEBUG nova.network.neutron [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Refreshing network info cache for port 4acc1eca-d65e-4e91-a872-24d025379936 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1843.174645] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance a7ed3eba-aced-4ff9-ba98-e29d44ef5c54 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1843.553760] env[62506]: DEBUG nova.network.neutron [-] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1843.624641] env[62506]: DEBUG nova.network.neutron [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1843.678582] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8fd213dd-d597-48c0-9387-19b4f5dd72ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1843.901031] env[62506]: DEBUG nova.network.neutron [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1844.055165] env[62506]: INFO nova.compute.manager [-] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Took 1.03 seconds to deallocate network for instance. [ 1844.058370] env[62506]: DEBUG nova.compute.claims [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1844.058370] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1844.181237] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e8433dc4-7c83-4288-acf4-5f5a2863592c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1844.401849] env[62506]: DEBUG oslo_concurrency.lockutils [req-a70fb760-cd08-4b68-969f-a731b45612ea req-a27d2112-da7d-4873-b148-971d33c213c0 service nova] Releasing lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1844.688274] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1844.851555] env[62506]: DEBUG nova.compute.manager [req-55c12a0c-ebdc-4257-af91-8d323f5baf2a req-5b2db873-35f3-4bfe-acd0-864aaebd5fca service nova] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Received event network-vif-deleted-4acc1eca-d65e-4e91-a872-24d025379936 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1845.193268] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7e185b7e-8a95-4931-96a8-1c27516f767d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1845.698947] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e0de75e0-ca8e-4525-ae7f-ed13fcd07338 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1846.204175] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 03e1ebe5-7a02-430e-b38c-d215e85d59ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1846.204175] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1846.204175] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1846.496085] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c08cf79-25cc-45c6-a1bb-17789112fd2c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1846.505173] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce60ab61-4123-4941-9bde-70f6d29c8183 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1846.539385] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-666fd866-49ee-4c5f-97ac-f923e92254fd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1846.551739] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccbf88c3-9fce-425d-93d8-43268d3d6d37 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1846.566855] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1847.070354] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1847.578517] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1847.578790] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 9.499s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1847.579803] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 25.559s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1848.460099] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5f41efd-b948-49c9-a2be-04602394ce09 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1848.469029] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aff78910-9195-4244-bdef-7b89387ea992 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1848.509564] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c3145df-a501-4cbe-9bb5-55d3308c40a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1848.517612] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa041b29-6ead-4f24-8b6e-4af4a9fa57f0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1848.532055] env[62506]: DEBUG nova.compute.provider_tree [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1848.576508] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1848.577162] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1849.035217] env[62506]: DEBUG nova.scheduler.client.report [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1849.084018] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1849.084064] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1849.084210] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1849.187689] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquiring lock "7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1849.187689] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Lock "7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1849.548459] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.969s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1849.549316] env[62506]: ERROR nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Traceback (most recent call last): [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self.driver.spawn(context, instance, image_meta, [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] vm_ref = self.build_virtual_machine(instance, [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] vif_infos = vmwarevif.get_vif_info(self._session, [ 1849.549316] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] for vif in network_info: [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return self._sync_wrapper(fn, *args, **kwargs) [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self.wait() [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self[:] = self._gt.wait() [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return self._exit_event.wait() [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] result = hub.switch() [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1849.549689] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return self.greenlet.switch() [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] result = function(*args, **kwargs) [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] return func(*args, **kwargs) [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] raise e [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] nwinfo = self.network_api.allocate_for_instance( [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] created_port_ids = self._update_ports_for_instance( [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] with excutils.save_and_reraise_exception(): [ 1849.549956] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] self.force_reraise() [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] raise self.value [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] updated_port = self._update_port( [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] _ensure_no_port_binding_failure(port) [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] raise exception.PortBindingFailed(port_id=port['id']) [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] nova.exception.PortBindingFailed: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. [ 1849.550234] env[62506]: ERROR nova.compute.manager [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] [ 1849.550507] env[62506]: DEBUG nova.compute.utils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1849.553706] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Build of instance 49423ca9-a31f-44d0-b05e-ab78b6b31321 was re-scheduled: Binding failed for port 3698a633-4ba4-43b5-8aef-9c4df07c9722, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1849.554098] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1849.554098] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquiring lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1849.554161] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Acquired lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1849.554332] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1849.555406] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 24.127s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1849.588772] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1849.589031] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1849.589210] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1849.589412] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b5527c82-048e-4738-8878-0807aa609596] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1849.589600] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1849.589790] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1849.593421] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1849.593964] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1849.593964] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1850.099769] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1850.288103] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1850.457219] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e6f03b8-6811-450d-bd4c-69ce550b65b9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1850.467164] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0579c9fc-53bb-4b1b-8ac6-ebdb5e0c3773 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1850.507596] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45e933ad-87f3-4a5a-a63f-f0b5af9fcda8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1850.515927] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1011d0cd-2fbf-42f6-85ee-c333dbb3e5fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1850.531235] env[62506]: DEBUG nova.compute.provider_tree [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1850.801641] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Releasing lock "refresh_cache-49423ca9-a31f-44d0-b05e-ab78b6b31321" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1850.801950] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1850.802134] env[62506]: DEBUG nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1850.802308] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1850.844096] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1851.035977] env[62506]: DEBUG nova.scheduler.client.report [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1851.347467] env[62506]: DEBUG nova.network.neutron [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1851.545259] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.988s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1851.545259] env[62506]: ERROR nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Traceback (most recent call last): [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self.driver.spawn(context, instance, image_meta, [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1851.545259] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] vm_ref = self.build_virtual_machine(instance, [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] vif_infos = vmwarevif.get_vif_info(self._session, [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] for vif in network_info: [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return self._sync_wrapper(fn, *args, **kwargs) [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self.wait() [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self[:] = self._gt.wait() [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return self._exit_event.wait() [ 1851.545516] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] result = hub.switch() [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return self.greenlet.switch() [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] result = function(*args, **kwargs) [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] return func(*args, **kwargs) [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] raise e [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] nwinfo = self.network_api.allocate_for_instance( [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1851.545820] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] created_port_ids = self._update_ports_for_instance( [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] with excutils.save_and_reraise_exception(): [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] self.force_reraise() [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] raise self.value [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] updated_port = self._update_port( [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] _ensure_no_port_binding_failure(port) [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1851.546115] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] raise exception.PortBindingFailed(port_id=port['id']) [ 1851.546377] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] nova.exception.PortBindingFailed: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. [ 1851.546377] env[62506]: ERROR nova.compute.manager [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] [ 1851.546377] env[62506]: DEBUG nova.compute.utils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1851.547207] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.881s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1851.549778] env[62506]: INFO nova.compute.claims [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1851.552764] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Build of instance 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0 was re-scheduled: Binding failed for port 43739195-3ff0-497a-871e-7dd5f4b9a3f9, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1851.553267] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1851.553498] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquiring lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1851.553672] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Acquired lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1851.554616] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1851.849762] env[62506]: INFO nova.compute.manager [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] [instance: 49423ca9-a31f-44d0-b05e-ab78b6b31321] Took 1.05 seconds to deallocate network for instance. [ 1851.967806] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "01ad7fbe-7ea0-425b-ba63-28537cb158b1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1851.967806] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "01ad7fbe-7ea0-425b-ba63-28537cb158b1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1852.084928] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1852.228968] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1852.735850] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Releasing lock "refresh_cache-4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1852.736100] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1852.737183] env[62506]: DEBUG nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1852.737183] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1852.811262] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1852.832397] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "854c955f-b327-4c25-87bd-3ad68dbff024" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1852.832397] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "854c955f-b327-4c25-87bd-3ad68dbff024" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1852.895565] env[62506]: INFO nova.scheduler.client.report [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Deleted allocations for instance 49423ca9-a31f-44d0-b05e-ab78b6b31321 [ 1852.997962] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-209377cf-ff85-4b94-adfb-3a7afa8b30f5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1853.006072] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e4bed3f-a24b-4d3f-95a7-9e6cb2094668 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1853.036517] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d63f56d5-40bd-4510-b2df-c605feee3555 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1853.043880] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78245f6b-7ce6-446e-a9a8-6e2f47f0d39c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1853.059204] env[62506]: DEBUG nova.compute.provider_tree [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1853.318321] env[62506]: DEBUG nova.network.neutron [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1853.416243] env[62506]: DEBUG oslo_concurrency.lockutils [None req-693137e1-86ed-4997-8152-adb568fb8a99 tempest-ImagesNegativeTestJSON-1152824057 tempest-ImagesNegativeTestJSON-1152824057-project-member] Lock "49423ca9-a31f-44d0-b05e-ab78b6b31321" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 52.080s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1853.562307] env[62506]: DEBUG nova.scheduler.client.report [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1853.821686] env[62506]: INFO nova.compute.manager [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] [instance: 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0] Took 1.08 seconds to deallocate network for instance. [ 1853.921856] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1854.070024] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.521s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1854.070024] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1854.074324] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 27.198s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1854.445887] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1854.581821] env[62506]: DEBUG nova.compute.utils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1854.596898] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1854.596898] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1854.751818] env[62506]: DEBUG nova.policy [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '52236c8ade5d475daef3fc79078fc85f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0d32c457d8894d3f84de575dca698f58', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1854.860016] env[62506]: INFO nova.scheduler.client.report [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Deleted allocations for instance 4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0 [ 1855.000498] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2de4dba2-0245-481f-b023-ee393ef1cd9f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1855.009178] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79edf84d-1b24-4862-b4b2-20a2eceb389f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1855.055125] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f084f93-b73f-42ae-807e-775fa17d85a0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1855.065268] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fb7a3d5-5fc0-4631-b0a8-10fb1bf2b7c0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1855.082365] env[62506]: DEBUG nova.compute.provider_tree [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1855.093995] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1855.367392] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa6bad71-3537-4444-a962-72d0d4dc396f tempest-TenantUsagesTestJSON-1822050169 tempest-TenantUsagesTestJSON-1822050169-project-member] Lock "4b0f2983-2bc8-43c9-8ff4-9f68bd55daf0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 52.262s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1855.585813] env[62506]: DEBUG nova.scheduler.client.report [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1855.872737] env[62506]: DEBUG nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1856.091529] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.017s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1856.092172] env[62506]: ERROR nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Traceback (most recent call last): [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self.driver.spawn(context, instance, image_meta, [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] vm_ref = self.build_virtual_machine(instance, [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] vif_infos = vmwarevif.get_vif_info(self._session, [ 1856.092172] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] for vif in network_info: [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return self._sync_wrapper(fn, *args, **kwargs) [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self.wait() [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self[:] = self._gt.wait() [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return self._exit_event.wait() [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] result = hub.switch() [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1856.092517] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return self.greenlet.switch() [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] result = function(*args, **kwargs) [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] return func(*args, **kwargs) [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] raise e [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] nwinfo = self.network_api.allocate_for_instance( [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] created_port_ids = self._update_ports_for_instance( [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] with excutils.save_and_reraise_exception(): [ 1856.092909] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] self.force_reraise() [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] raise self.value [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] updated_port = self._update_port( [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] _ensure_no_port_binding_failure(port) [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] raise exception.PortBindingFailed(port_id=port['id']) [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] nova.exception.PortBindingFailed: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. [ 1856.093277] env[62506]: ERROR nova.compute.manager [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] [ 1856.093569] env[62506]: DEBUG nova.compute.utils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1856.097899] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Build of instance cb3b9b8c-4a27-430d-8653-f1afe3ac52cd was re-scheduled: Binding failed for port 1e2672bc-6822-4ac5-a664-248ad0cd6bb6, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1856.099029] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1856.099029] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquiring lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1856.099029] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Acquired lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1856.099029] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1856.100513] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.567s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1856.102173] env[62506]: INFO nova.compute.claims [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1856.107265] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Successfully created port: cd070db2-3b79-4455-bcb8-7cdca2b63e18 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1856.116228] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1856.150137] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1856.150848] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1856.150848] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1856.150848] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1856.151035] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1856.151181] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1856.151401] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1856.151563] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1856.151729] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1856.151952] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1856.152176] env[62506]: DEBUG nova.virt.hardware [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1856.153601] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-193e1e07-66aa-46e6-8b4a-5f62f92472aa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1856.164805] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f028d9fd-1964-477e-a046-649198ed21a0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1856.395563] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1856.658114] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1856.905351] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1857.406677] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Releasing lock "refresh_cache-cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1857.407282] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1857.407677] env[62506]: DEBUG nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1857.409375] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1857.493776] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1857.527113] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6a6a27a-b60b-4c3f-a258-5561939a530d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1857.539113] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9c9f91b-3faf-447d-97d1-518f0eff04cf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1857.574204] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82d4e56b-204c-40c4-bb2c-9b8a5eaa066d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1857.583067] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b465bcb-dac8-4a00-989e-f940536db42e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1857.598665] env[62506]: DEBUG nova.compute.provider_tree [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1858.000999] env[62506]: DEBUG nova.network.neutron [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1858.103201] env[62506]: DEBUG nova.scheduler.client.report [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1858.507285] env[62506]: INFO nova.compute.manager [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] [instance: cb3b9b8c-4a27-430d-8653-f1afe3ac52cd] Took 1.10 seconds to deallocate network for instance. [ 1858.613054] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.512s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1858.613705] env[62506]: DEBUG nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1858.618909] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.764s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1858.818430] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "6cdba5b2-af96-4766-9a97-3a758983c49b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1858.818703] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "6cdba5b2-af96-4766-9a97-3a758983c49b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1859.126566] env[62506]: DEBUG nova.compute.utils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1859.136480] env[62506]: DEBUG nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 1859.552016] env[62506]: INFO nova.scheduler.client.report [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Deleted allocations for instance cb3b9b8c-4a27-430d-8653-f1afe3ac52cd [ 1859.556291] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5cc6bc4-b0fe-4cf4-9e82-3ad8214c7bfc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.564861] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fab8cfc-5d90-4b06-8bf6-58f7c069402c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.604536] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05a1dff7-baf6-4320-abfe-29ec946424cd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.612842] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1859.615610] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0ce6d87-8aad-4c8f-8584-5d670bfe35d3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.631236] env[62506]: DEBUG nova.compute.provider_tree [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1859.636596] env[62506]: DEBUG nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1859.751287] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquiring lock "08cc6c54-48f2-4064-93a4-4debb98ab606" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1859.751541] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Lock "08cc6c54-48f2-4064-93a4-4debb98ab606" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1860.061943] env[62506]: DEBUG oslo_concurrency.lockutils [None req-a37cf49b-4fb2-4034-a82d-dd331c8b02f0 tempest-ServersAdminNegativeTestJSON-1793915023 tempest-ServersAdminNegativeTestJSON-1793915023-project-member] Lock "cb3b9b8c-4a27-430d-8653-f1afe3ac52cd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 56.413s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1860.136375] env[62506]: DEBUG nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1860.262388] env[62506]: ERROR nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1860.262388] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1860.262388] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1860.262388] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1860.262388] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1860.262388] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1860.262388] env[62506]: ERROR nova.compute.manager raise self.value [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1860.262388] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1860.262388] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1860.262388] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1860.262783] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1860.262783] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1860.262783] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1860.262783] env[62506]: ERROR nova.compute.manager [ 1860.262783] env[62506]: Traceback (most recent call last): [ 1860.262783] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1860.262783] env[62506]: listener.cb(fileno) [ 1860.262783] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1860.262783] env[62506]: result = function(*args, **kwargs) [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1860.262783] env[62506]: return func(*args, **kwargs) [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1860.262783] env[62506]: raise e [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1860.262783] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1860.262783] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1860.262783] env[62506]: with excutils.save_and_reraise_exception(): [ 1860.262783] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1860.262783] env[62506]: self.force_reraise() [ 1860.262783] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1860.262783] env[62506]: raise self.value [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1860.262783] env[62506]: updated_port = self._update_port( [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1860.262783] env[62506]: _ensure_no_port_binding_failure(port) [ 1860.262783] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1860.262783] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1860.263526] env[62506]: nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1860.263526] env[62506]: Removing descriptor: 15 [ 1860.263526] env[62506]: ERROR nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Traceback (most recent call last): [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] yield resources [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self.driver.spawn(context, instance, image_meta, [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1860.263526] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] vm_ref = self.build_virtual_machine(instance, [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] vif_infos = vmwarevif.get_vif_info(self._session, [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] for vif in network_info: [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return self._sync_wrapper(fn, *args, **kwargs) [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self.wait() [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self[:] = self._gt.wait() [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return self._exit_event.wait() [ 1860.263849] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] result = hub.switch() [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return self.greenlet.switch() [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] result = function(*args, **kwargs) [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return func(*args, **kwargs) [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] raise e [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] nwinfo = self.network_api.allocate_for_instance( [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1860.264216] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] created_port_ids = self._update_ports_for_instance( [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] with excutils.save_and_reraise_exception(): [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self.force_reraise() [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] raise self.value [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] updated_port = self._update_port( [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] _ensure_no_port_binding_failure(port) [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1860.264525] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] raise exception.PortBindingFailed(port_id=port['id']) [ 1860.264867] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1860.264867] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] [ 1860.264867] env[62506]: INFO nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Terminating instance [ 1860.514325] env[62506]: DEBUG nova.compute.manager [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Received event network-changed-cd070db2-3b79-4455-bcb8-7cdca2b63e18 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1860.514540] env[62506]: DEBUG nova.compute.manager [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Refreshing instance network info cache due to event network-changed-cd070db2-3b79-4455-bcb8-7cdca2b63e18. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1860.514764] env[62506]: DEBUG oslo_concurrency.lockutils [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] Acquiring lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1860.514910] env[62506]: DEBUG oslo_concurrency.lockutils [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] Acquired lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1860.516288] env[62506]: DEBUG nova.network.neutron [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Refreshing network info cache for port cd070db2-3b79-4455-bcb8-7cdca2b63e18 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1860.565917] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1860.651638] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.033s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1860.653230] env[62506]: ERROR nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Traceback (most recent call last): [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self.driver.spawn(context, instance, image_meta, [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] vm_ref = self.build_virtual_machine(instance, [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1860.653230] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] for vif in network_info: [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return self._sync_wrapper(fn, *args, **kwargs) [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self.wait() [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self[:] = self._gt.wait() [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return self._exit_event.wait() [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] result = hub.switch() [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1860.653497] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return self.greenlet.switch() [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] result = function(*args, **kwargs) [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] return func(*args, **kwargs) [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] raise e [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] nwinfo = self.network_api.allocate_for_instance( [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] created_port_ids = self._update_ports_for_instance( [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] with excutils.save_and_reraise_exception(): [ 1860.653784] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] self.force_reraise() [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] raise self.value [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] updated_port = self._update_port( [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] _ensure_no_port_binding_failure(port) [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] raise exception.PortBindingFailed(port_id=port['id']) [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] nova.exception.PortBindingFailed: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. [ 1860.654097] env[62506]: ERROR nova.compute.manager [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] [ 1860.654338] env[62506]: DEBUG nova.compute.utils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1860.657231] env[62506]: DEBUG nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1860.660432] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.370s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1860.663098] env[62506]: INFO nova.compute.claims [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1860.672629] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Build of instance 4a95ce6c-d1c0-4677-a330-70620a05989d was re-scheduled: Binding failed for port a0aeea44-cf68-4788-ae5d-b1eef63e32c2, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1860.672629] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1860.672629] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquiring lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1860.672629] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Acquired lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1860.673227] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1860.708864] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1860.708864] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1860.709475] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1860.709475] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1860.709630] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1860.709823] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1860.710118] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1860.710399] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1860.710585] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1860.710703] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1860.710909] env[62506]: DEBUG nova.virt.hardware [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1860.711796] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d1291a4-84c9-46d1-bad2-d6f0738b09ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.721248] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-840e3831-6136-4ed1-b955-7a7c470e6398 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.744319] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1860.750960] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Creating folder: Project (e2cb12a7aaef4a0b816237ba33bd9da4). Parent ref: group-v446953. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1860.752667] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-01e54652-893f-4e96-9c92-744bd66cffa2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.763160] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Created folder: Project (e2cb12a7aaef4a0b816237ba33bd9da4) in parent group-v446953. [ 1860.763390] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Creating folder: Instances. Parent ref: group-v446959. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1860.763654] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5247b70f-5b2e-41f3-9aa5-f78d55f72159 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.768255] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquiring lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1860.772345] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Created folder: Instances in parent group-v446959. [ 1860.772547] env[62506]: DEBUG oslo.service.loopingcall [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1860.772730] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 1860.772935] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-6ad6d9d5-4219-4412-ab67-81c7afb02e0f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.796599] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1860.796599] env[62506]: value = "task-2190786" [ 1860.796599] env[62506]: _type = "Task" [ 1860.796599] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1860.830983] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquiring lock "3d034f40-5896-49bc-bed1-9062e1fde1e3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1860.831346] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Lock "3d034f40-5896-49bc-bed1-9062e1fde1e3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.071137] env[62506]: DEBUG nova.network.neutron [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1861.101909] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1861.216114] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1861.311451] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190786, 'name': CreateVM_Task, 'duration_secs': 0.378236} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1861.311774] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 1861.312337] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1861.312721] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1861.314372] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1861.314804] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-671332d2-a282-41f1-b079-1bfe7c357bd4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.321154] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1861.321154] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52aeb05c-121b-f322-54d4-0f9983666603" [ 1861.321154] env[62506]: _type = "Task" [ 1861.321154] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1861.333783] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52aeb05c-121b-f322-54d4-0f9983666603, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1861.426154] env[62506]: DEBUG nova.network.neutron [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1861.542030] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1861.710036] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquiring lock "b5d6b647-e24f-4e53-b448-0614334fd760" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1861.710036] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Lock "b5d6b647-e24f-4e53-b448-0614334fd760" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.841123] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52aeb05c-121b-f322-54d4-0f9983666603, 'name': SearchDatastore_Task, 'duration_secs': 0.022116} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1861.844534] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1861.844780] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1861.845040] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1861.845185] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1861.845914] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1861.846834] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d0e9fb55-6885-4519-947a-b1436d476c4b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.857806] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1861.858392] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 1861.858756] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-23ef9951-31ca-47ab-b449-c70066534101 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.866085] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1861.866085] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52ec61e2-b082-73e5-b802-a03a73f0587e" [ 1861.866085] env[62506]: _type = "Task" [ 1861.866085] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1861.881152] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52ec61e2-b082-73e5-b802-a03a73f0587e, 'name': SearchDatastore_Task, 'duration_secs': 0.009953} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1861.881985] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ecfa3293-3c2f-4751-8477-fb233f072f6c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.886940] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1861.886940] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52a8fd42-1a93-538c-64ea-376675082dc8" [ 1861.886940] env[62506]: _type = "Task" [ 1861.886940] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1861.901089] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52a8fd42-1a93-538c-64ea-376675082dc8, 'name': SearchDatastore_Task, 'duration_secs': 0.00878} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1861.901962] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1861.901962] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 1861.901962] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e3207d0b-997b-4f13-9299-179edeec6f59 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.910833] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1861.910833] env[62506]: value = "task-2190787" [ 1861.910833] env[62506]: _type = "Task" [ 1861.910833] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1861.920597] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190787, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1861.930117] env[62506]: DEBUG oslo_concurrency.lockutils [req-c5579aaa-11fb-4199-8a61-9d14c49a9983 req-40bc4a87-7262-459b-a8e5-e75579c429a3 service nova] Releasing lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1861.930117] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquired lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1861.930117] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1862.042136] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Releasing lock "refresh_cache-4a95ce6c-d1c0-4677-a330-70620a05989d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1862.042409] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1862.042680] env[62506]: DEBUG nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1862.042791] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1862.097307] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1862.186832] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc1aab2e-393b-433c-ad1b-29fdd427cdc8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.197045] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-406ea450-4f69-4756-9c7b-1cadd7a0aa04 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.233291] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-469f448f-1d73-434f-a126-7c0d13481b95 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.243589] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4a7d9e0-b35c-47f6-bd20-933a5cd55f4c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.260364] env[62506]: DEBUG nova.compute.provider_tree [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1862.427206] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190787, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.488591} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1862.427206] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 1862.429301] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 1862.429301] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-3df008ad-146b-46b4-b5e8-3722fce41b79 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.439638] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1862.439638] env[62506]: value = "task-2190788" [ 1862.439638] env[62506]: _type = "Task" [ 1862.439638] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1862.449159] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190788, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1862.468776] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1862.603132] env[62506]: DEBUG nova.network.neutron [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1862.697029] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1862.769024] env[62506]: DEBUG nova.scheduler.client.report [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1862.949390] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190788, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.236805} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1862.949624] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 1862.950422] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2e18ecb-cec1-4797-8fac-bf06c6cc58f2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.970790] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Reconfiguring VM instance instance-00000010 to attach disk [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 1862.971419] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-7b36c3f9-93fd-438d-9b5c-b435d36ef1b4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.992537] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1862.992537] env[62506]: value = "task-2190789" [ 1862.992537] env[62506]: _type = "Task" [ 1862.992537] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1863.002779] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190789, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1863.105397] env[62506]: INFO nova.compute.manager [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] [instance: 4a95ce6c-d1c0-4677-a330-70620a05989d] Took 1.06 seconds to deallocate network for instance. [ 1863.201026] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Releasing lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1863.201571] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1863.201814] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1863.202431] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9b90e95f-8b24-4da4-99fd-f873b5c5e4fe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.208681] env[62506]: DEBUG nova.compute.manager [req-05eef3bd-2df5-4d0f-aba6-6999d9fa9c0a req-665651af-e252-4381-93e2-a7ed65fc354f service nova] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Received event network-vif-deleted-cd070db2-3b79-4455-bcb8-7cdca2b63e18 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1863.216436] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-283126e5-186a-4938-ac0c-812774546a96 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.241282] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 386935e4-c2b7-4207-9670-cd532ff12f70 could not be found. [ 1863.241568] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1863.241796] env[62506]: INFO nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1863.242092] env[62506]: DEBUG oslo.service.loopingcall [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1863.242350] env[62506]: DEBUG nova.compute.manager [-] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1863.242481] env[62506]: DEBUG nova.network.neutron [-] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1863.273186] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.612s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1863.273517] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1863.276271] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.711s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1863.437528] env[62506]: DEBUG nova.network.neutron [-] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1863.506632] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190789, 'name': ReconfigVM_Task, 'duration_secs': 0.39585} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1863.509481] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Reconfigured VM instance instance-00000010 to attach disk [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1863.509481] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-de641067-f729-4a2c-9b1e-eb1ea4122165 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.514396] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1863.514396] env[62506]: value = "task-2190790" [ 1863.514396] env[62506]: _type = "Task" [ 1863.514396] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1863.524405] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190790, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1863.610613] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1863.646150] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquiring lock "dcd5a800-1a28-45b3-8eff-a2e8cb871491" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1863.646150] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Lock "dcd5a800-1a28-45b3-8eff-a2e8cb871491" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1863.781967] env[62506]: DEBUG nova.compute.utils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1863.793883] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1863.794094] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1863.942178] env[62506]: DEBUG nova.network.neutron [-] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1863.955130] env[62506]: DEBUG nova.policy [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd3d4aed209734b54a5860f96368c620d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5120d07a69a240c0b71da679d98063b1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1864.030179] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190790, 'name': Rename_Task, 'duration_secs': 0.132743} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1864.030179] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 1864.030179] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-af5e7173-385d-4a58-95af-1a6f8b2b60c6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1864.039098] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1864.039098] env[62506]: value = "task-2190791" [ 1864.039098] env[62506]: _type = "Task" [ 1864.039098] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1864.048402] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190791, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1864.147263] env[62506]: INFO nova.scheduler.client.report [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Deleted allocations for instance 4a95ce6c-d1c0-4677-a330-70620a05989d [ 1864.200465] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5266a1a-684b-4873-bf0f-03e4144ebc95 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1864.208915] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d24c727c-f486-44ab-b906-328c3a2ff1d6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1865.002630] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1865.004864] env[62506]: INFO nova.compute.manager [-] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Took 1.76 seconds to deallocate network for instance. [ 1865.005334] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1afe92a7-5786-490d-b815-bb29da3f1889 tempest-ServersWithSpecificFlavorTestJSON-111680658 tempest-ServersWithSpecificFlavorTestJSON-111680658-project-member] Lock "4a95ce6c-d1c0-4677-a330-70620a05989d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 59.985s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1865.007675] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Successfully created port: 285501f8-43f4-493c-85d9-9c87ba6dd1f8 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1865.017488] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8140ddf-8994-42fa-b1db-17b779890b42 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1865.021209] env[62506]: DEBUG nova.compute.claims [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1865.021209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1865.021669] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "04218b42-91dd-4045-8a40-f8829910e064" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1865.021704] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "04218b42-91dd-4045-8a40-f8829910e064" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1865.031728] env[62506]: DEBUG oslo_vmware.api [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190791, 'name': PowerOnVM_Task, 'duration_secs': 0.478432} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1865.033501] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5be6cfb-1297-451e-a6da-47e277d65e68 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1865.039052] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 1865.039181] env[62506]: INFO nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Took 4.38 seconds to spawn the instance on the hypervisor. [ 1865.039362] env[62506]: DEBUG nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1865.040808] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfacc5bb-a2c8-4b3a-8439-5df666be5df9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1865.055080] env[62506]: DEBUG nova.compute.provider_tree [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1865.522572] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1865.575895] env[62506]: INFO nova.compute.manager [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Took 38.07 seconds to build instance. [ 1865.586049] env[62506]: ERROR nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [req-dc87d01a-5d49-4268-b758-cbd4b787f659] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-dc87d01a-5d49-4268-b758-cbd4b787f659"}]}: nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1865.603992] env[62506]: DEBUG nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1865.622504] env[62506]: DEBUG nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1865.622749] env[62506]: DEBUG nova.compute.provider_tree [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1865.636994] env[62506]: DEBUG nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1865.658338] env[62506]: DEBUG nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1866.038306] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1866.053019] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1866.067019] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1866.067019] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1866.067019] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1866.067272] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1866.067272] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1866.067272] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1866.067272] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1866.067272] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1866.067410] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1866.067410] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1866.067410] env[62506]: DEBUG nova.virt.hardware [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1866.067962] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cd8c898-737f-474c-8f48-3651c631aa5c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.078415] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d274f807-c233-4c49-ac45-fd438b7cb043 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.084085] env[62506]: DEBUG oslo_concurrency.lockutils [None req-58a532b4-b636-4da4-8e75-3a499be11510 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "af042cab-9132-4425-b017-133ec85afa0c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 54.199s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1866.084490] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d62b144-9740-4659-930c-a8199acf4281 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.103915] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b063f2e-0f54-4fdb-aac7-5e2932992160 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.134835] env[62506]: INFO nova.compute.manager [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Rebuilding instance [ 1866.138136] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29b7166d-ee75-435e-b488-e97efbcaed5c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.146500] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fe8c05b-5e98-416f-b24d-c0d76b835ca6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.164620] env[62506]: DEBUG nova.compute.provider_tree [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1866.190296] env[62506]: DEBUG nova.compute.manager [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1866.190656] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0af8fc0-317a-440e-9c21-2849502013c2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.590244] env[62506]: DEBUG nova.compute.manager [None req-7031ddb5-fa9c-44c4-b283-dd8e7fd85301 tempest-ServersListShow296Test-547794886 tempest-ServersListShow296Test-547794886-project-member] [instance: c168f33c-ab2a-4d6c-900d-9831d0c034fa] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1866.716446] env[62506]: DEBUG nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 47 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1866.717909] env[62506]: DEBUG nova.compute.provider_tree [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 47 to 48 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1866.718217] env[62506]: DEBUG nova.compute.provider_tree [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1867.062113] env[62506]: DEBUG nova.compute.manager [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Received event network-changed-285501f8-43f4-493c-85d9-9c87ba6dd1f8 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1867.062381] env[62506]: DEBUG nova.compute.manager [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Refreshing instance network info cache due to event network-changed-285501f8-43f4-493c-85d9-9c87ba6dd1f8. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1867.062577] env[62506]: DEBUG oslo_concurrency.lockutils [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] Acquiring lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1867.062777] env[62506]: DEBUG oslo_concurrency.lockutils [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] Acquired lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1867.062997] env[62506]: DEBUG nova.network.neutron [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Refreshing network info cache for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1867.094611] env[62506]: DEBUG nova.compute.manager [None req-7031ddb5-fa9c-44c4-b283-dd8e7fd85301 tempest-ServersListShow296Test-547794886 tempest-ServersListShow296Test-547794886-project-member] [instance: c168f33c-ab2a-4d6c-900d-9831d0c034fa] Instance disappeared before build. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2446}} [ 1867.199306] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquiring lock "ecc8aab4-a172-4218-8b09-ccfde0d03903" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1867.199853] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Lock "ecc8aab4-a172-4218-8b09-ccfde0d03903" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1867.211874] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 1867.212218] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-0a25b79f-1788-4f0b-9830-df81a0afeb92 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1867.219798] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1867.219798] env[62506]: value = "task-2190792" [ 1867.219798] env[62506]: _type = "Task" [ 1867.219798] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1867.225016] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 3.949s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1867.225647] env[62506]: ERROR nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] Traceback (most recent call last): [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self.driver.spawn(context, instance, image_meta, [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] vm_ref = self.build_virtual_machine(instance, [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] vif_infos = vmwarevif.get_vif_info(self._session, [ 1867.225647] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] for vif in network_info: [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return self._sync_wrapper(fn, *args, **kwargs) [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self.wait() [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self[:] = self._gt.wait() [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return self._exit_event.wait() [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] result = hub.switch() [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1867.226190] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return self.greenlet.switch() [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] result = function(*args, **kwargs) [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] return func(*args, **kwargs) [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] raise e [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] nwinfo = self.network_api.allocate_for_instance( [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] created_port_ids = self._update_ports_for_instance( [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] with excutils.save_and_reraise_exception(): [ 1867.226518] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] self.force_reraise() [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] raise self.value [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] updated_port = self._update_port( [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] _ensure_no_port_binding_failure(port) [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] raise exception.PortBindingFailed(port_id=port['id']) [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] nova.exception.PortBindingFailed: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. [ 1867.226847] env[62506]: ERROR nova.compute.manager [instance: b5527c82-048e-4738-8878-0807aa609596] [ 1867.227123] env[62506]: DEBUG nova.compute.utils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1867.227869] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.372s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1867.229945] env[62506]: INFO nova.compute.claims [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1867.235874] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Build of instance b5527c82-048e-4738-8878-0807aa609596 was re-scheduled: Binding failed for port 812e694a-b615-422a-bfb7-829c81f957ff, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1867.236427] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1867.236657] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1867.236803] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1867.236958] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1867.242262] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190792, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1867.261258] env[62506]: ERROR nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1867.261258] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1867.261258] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1867.261258] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1867.261258] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1867.261258] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1867.261258] env[62506]: ERROR nova.compute.manager raise self.value [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1867.261258] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1867.261258] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1867.261258] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1867.261671] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1867.261671] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1867.261671] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1867.261671] env[62506]: ERROR nova.compute.manager [ 1867.261671] env[62506]: Traceback (most recent call last): [ 1867.261671] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1867.261671] env[62506]: listener.cb(fileno) [ 1867.261671] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1867.261671] env[62506]: result = function(*args, **kwargs) [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1867.261671] env[62506]: return func(*args, **kwargs) [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1867.261671] env[62506]: raise e [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1867.261671] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1867.261671] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1867.261671] env[62506]: with excutils.save_and_reraise_exception(): [ 1867.261671] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1867.261671] env[62506]: self.force_reraise() [ 1867.261671] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1867.261671] env[62506]: raise self.value [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1867.261671] env[62506]: updated_port = self._update_port( [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1867.261671] env[62506]: _ensure_no_port_binding_failure(port) [ 1867.261671] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1867.261671] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1867.262302] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1867.262302] env[62506]: Removing descriptor: 19 [ 1867.262302] env[62506]: ERROR nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Traceback (most recent call last): [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] yield resources [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self.driver.spawn(context, instance, image_meta, [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1867.262302] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] vm_ref = self.build_virtual_machine(instance, [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] vif_infos = vmwarevif.get_vif_info(self._session, [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] for vif in network_info: [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return self._sync_wrapper(fn, *args, **kwargs) [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self.wait() [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self[:] = self._gt.wait() [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return self._exit_event.wait() [ 1867.262616] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] result = hub.switch() [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return self.greenlet.switch() [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] result = function(*args, **kwargs) [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return func(*args, **kwargs) [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] raise e [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] nwinfo = self.network_api.allocate_for_instance( [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1867.262955] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] created_port_ids = self._update_ports_for_instance( [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] with excutils.save_and_reraise_exception(): [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self.force_reraise() [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] raise self.value [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] updated_port = self._update_port( [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] _ensure_no_port_binding_failure(port) [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1867.263338] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] raise exception.PortBindingFailed(port_id=port['id']) [ 1867.263595] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1867.263595] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] [ 1867.263595] env[62506]: INFO nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Terminating instance [ 1867.611285] env[62506]: DEBUG nova.network.neutron [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1867.613047] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1867.613201] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1867.617128] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7031ddb5-fa9c-44c4-b283-dd8e7fd85301 tempest-ServersListShow296Test-547794886 tempest-ServersListShow296Test-547794886-project-member] Lock "c168f33c-ab2a-4d6c-900d-9831d0c034fa" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 44.062s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1867.732917] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190792, 'name': PowerOffVM_Task, 'duration_secs': 0.118915} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1867.733430] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 1867.734175] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1867.735326] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92f0528e-7923-413e-a5e0-63fd466e89a0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1867.744744] env[62506]: DEBUG nova.network.neutron [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1867.755432] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 1867.756224] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d7d43281-2a92-4b3c-8caa-09fdedd09363 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1867.773023] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1867.779034] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1867.789029] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 1867.789029] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 1867.789029] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Deleting the datastore file [datastore2] af042cab-9132-4425-b017-133ec85afa0c {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1867.789029] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-782a5691-ebf8-4773-abbc-70c33f6c923e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1867.796173] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1867.796173] env[62506]: value = "task-2190794" [ 1867.796173] env[62506]: _type = "Task" [ 1867.796173] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1867.807378] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190794, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1868.021950] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1868.120563] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1868.252355] env[62506]: DEBUG oslo_concurrency.lockutils [req-ff0d1ad0-5174-4bfb-a30a-ed0d89312f32 req-16a2ec80-ddb9-4880-a253-d6e52d4fbd04 service nova] Releasing lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1868.256102] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquired lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1868.256302] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1868.309431] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190794, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.093195} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1868.311977] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1868.312189] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 1868.312367] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1868.526574] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-b5527c82-048e-4738-8878-0807aa609596" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1868.526574] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1868.526574] env[62506]: DEBUG nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1868.526574] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1868.543027] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1868.609976] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1868.610323] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1868.652898] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1868.691551] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-652b8698-9ff5-47c3-a49b-351c030cf78c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1868.699357] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-638c7994-ad90-4a42-9efd-f6d81f3706b2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1868.728391] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-905d01d7-affc-49b7-8691-034b78af46df {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1868.736119] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f8d8e3f-ab96-4761-9b1a-bc4c7172e4d5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1868.750959] env[62506]: DEBUG nova.compute.provider_tree [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1868.801446] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1868.925304] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1868.990160] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquiring lock "19afb920-58b0-4ad9-be99-d0a8e930c680" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1868.990562] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Lock "19afb920-58b0-4ad9-be99-d0a8e930c680" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1869.045910] env[62506]: DEBUG nova.network.neutron [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1869.117188] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1869.128511] env[62506]: DEBUG nova.compute.manager [req-9f78e83e-bb6e-4383-9600-ef69a0eec813 req-fa917bfa-726d-4cd5-86e1-cbda0864c925 service nova] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Received event network-vif-deleted-285501f8-43f4-493c-85d9-9c87ba6dd1f8 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1869.272967] env[62506]: ERROR nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [req-a8567474-e158-4a1b-a9db-4c5fd1439829] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-a8567474-e158-4a1b-a9db-4c5fd1439829"}]} [ 1869.289181] env[62506]: DEBUG nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1869.303707] env[62506]: DEBUG nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1869.304590] env[62506]: DEBUG nova.compute.provider_tree [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1869.316154] env[62506]: DEBUG nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1869.342214] env[62506]: DEBUG nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1869.352346] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1869.352591] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1869.352749] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1869.352943] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1869.353108] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1869.353260] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1869.353466] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1869.353622] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1869.353781] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1869.353945] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1869.354131] env[62506]: DEBUG nova.virt.hardware [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1869.354981] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4987f4e6-bbe6-417c-b3e0-820f1792f1c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.363199] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-244c0715-906c-400c-bc4b-390871818b18 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.380617] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1869.386780] env[62506]: DEBUG oslo.service.loopingcall [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1869.389193] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 1869.389575] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3b4cfe9d-ab34-44e9-b436-35413ac77218 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.409431] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1869.409431] env[62506]: value = "task-2190795" [ 1869.409431] env[62506]: _type = "Task" [ 1869.409431] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1869.419352] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190795, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1869.429442] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Releasing lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1869.429884] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1869.430091] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1869.430361] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5e52f391-2c22-4975-a5f0-d2943dd99e83 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.441745] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c1f5563-f997-4156-a35b-bc03110e1298 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.467116] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 88322331-7cb9-46a9-8e8c-82f0198c90d8 could not be found. [ 1869.467374] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1869.467645] env[62506]: INFO nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1869.467852] env[62506]: DEBUG oslo.service.loopingcall [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1869.470573] env[62506]: DEBUG nova.compute.manager [-] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1869.470672] env[62506]: DEBUG nova.network.neutron [-] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1869.495074] env[62506]: DEBUG nova.network.neutron [-] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1869.551110] env[62506]: INFO nova.compute.manager [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: b5527c82-048e-4738-8878-0807aa609596] Took 1.03 seconds to deallocate network for instance. [ 1869.796175] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58a04a78-be3e-4a2e-bd51-4f619574fb65 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.804781] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-548ffa31-2ad9-434e-9f61-00b75723e7d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.839046] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aecf5f46-9342-409e-af98-59746ac39e47 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.843920] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3085871f-6727-4a74-8d90-aca1cedbd007 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.858903] env[62506]: DEBUG nova.compute.provider_tree [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1869.919950] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190795, 'name': CreateVM_Task, 'duration_secs': 0.265168} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1869.920150] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 1869.920559] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1869.920711] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1869.921072] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1869.921306] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3eb2026c-70f7-487c-9c5d-fd9e8ff70b76 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.925664] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1869.925664] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]524984fc-b2e3-a87e-a248-5a7b298cf762" [ 1869.925664] env[62506]: _type = "Task" [ 1869.925664] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1869.933184] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]524984fc-b2e3-a87e-a248-5a7b298cf762, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1869.998104] env[62506]: DEBUG nova.network.neutron [-] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1870.389695] env[62506]: DEBUG nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 50 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1870.390052] env[62506]: DEBUG nova.compute.provider_tree [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 50 to 51 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1870.390250] env[62506]: DEBUG nova.compute.provider_tree [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1870.436027] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]524984fc-b2e3-a87e-a248-5a7b298cf762, 'name': SearchDatastore_Task, 'duration_secs': 0.015004} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1870.436140] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1870.436365] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1870.436593] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1870.436913] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1870.436913] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1870.437194] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0295bfcf-1624-4eda-9d68-38df34d92a12 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1870.444714] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1870.444953] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 1870.445566] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-dbbcd5c8-1a52-46dc-8487-cce48cdb064b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1870.450298] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1870.450298] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]520d3d5e-4cbc-6d22-2feb-a38e0ac49f39" [ 1870.450298] env[62506]: _type = "Task" [ 1870.450298] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1870.457692] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]520d3d5e-4cbc-6d22-2feb-a38e0ac49f39, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1870.504827] env[62506]: INFO nova.compute.manager [-] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Took 1.03 seconds to deallocate network for instance. [ 1870.507179] env[62506]: DEBUG nova.compute.claims [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1870.507363] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1870.580855] env[62506]: INFO nova.scheduler.client.report [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Deleted allocations for instance b5527c82-048e-4738-8878-0807aa609596 [ 1870.895984] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.668s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1870.897327] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1870.903352] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.846s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1870.963529] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]520d3d5e-4cbc-6d22-2feb-a38e0ac49f39, 'name': SearchDatastore_Task, 'duration_secs': 0.007407} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1870.965013] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-436bd046-e02b-4931-9fad-c2efb2238e3f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1870.970126] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1870.970126] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52ca1104-91e2-8795-6c02-53aaa3bf6004" [ 1870.970126] env[62506]: _type = "Task" [ 1870.970126] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1870.978612] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52ca1104-91e2-8795-6c02-53aaa3bf6004, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1871.091226] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7f1753de-68dd-4f37-b0f6-3901ec69b5a3 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "b5527c82-048e-4738-8878-0807aa609596" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 62.453s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1871.410494] env[62506]: DEBUG nova.compute.utils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1871.415977] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1871.416157] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1871.481881] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52ca1104-91e2-8795-6c02-53aaa3bf6004, 'name': SearchDatastore_Task, 'duration_secs': 0.008515} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1871.484337] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1871.484607] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 1871.485452] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-ef3ff64d-7ba5-4b41-93c2-e3848175fd15 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.491860] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1871.491860] env[62506]: value = "task-2190796" [ 1871.491860] env[62506]: _type = "Task" [ 1871.491860] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1871.502369] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190796, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1871.512659] env[62506]: DEBUG nova.policy [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '58daf167c8ae4487b0be7474226b0893', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd1e253d54c83479aac8f17be9283898d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1871.597647] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1871.871084] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81b7d347-6b14-4489-9d31-ae0b7560ec13 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.878949] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ff4635c-65a0-4c6f-98ab-5490ee44b3f9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.914409] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8da896bb-a171-45c7-a16c-66d412356478 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.919191] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1871.926190] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14288045-ac67-452c-af0d-c65c194365d4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1871.942203] env[62506]: DEBUG nova.compute.provider_tree [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1872.002948] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190796, 'name': CopyVirtualDisk_Task} progress is 51%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1872.091118] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Successfully created port: db27c24c-5cdb-4c11-b330-073ad4c46df0 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1872.118737] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1872.364573] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "dc1c594f-7625-495a-8948-cee046bf204a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1872.364859] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "dc1c594f-7625-495a-8948-cee046bf204a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1872.445588] env[62506]: DEBUG nova.scheduler.client.report [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1872.503122] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190796, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.606855} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1872.503401] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 1872.503616] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 1872.503856] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-670d32d0-c108-4f45-9871-a0e6e24e0ba5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.510588] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1872.510588] env[62506]: value = "task-2190797" [ 1872.510588] env[62506]: _type = "Task" [ 1872.510588] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1872.519699] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190797, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1872.931349] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1872.950844] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.047s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1872.951566] env[62506]: ERROR nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Traceback (most recent call last): [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self.driver.spawn(context, instance, image_meta, [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] vm_ref = self.build_virtual_machine(instance, [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] vif_infos = vmwarevif.get_vif_info(self._session, [ 1872.951566] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] for vif in network_info: [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return self._sync_wrapper(fn, *args, **kwargs) [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self.wait() [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self[:] = self._gt.wait() [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return self._exit_event.wait() [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] result = hub.switch() [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1872.951846] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return self.greenlet.switch() [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] result = function(*args, **kwargs) [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] return func(*args, **kwargs) [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] raise e [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] nwinfo = self.network_api.allocate_for_instance( [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] created_port_ids = self._update_ports_for_instance( [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] with excutils.save_and_reraise_exception(): [ 1872.952308] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] self.force_reraise() [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] raise self.value [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] updated_port = self._update_port( [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] _ensure_no_port_binding_failure(port) [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] raise exception.PortBindingFailed(port_id=port['id']) [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] nova.exception.PortBindingFailed: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. [ 1872.952609] env[62506]: ERROR nova.compute.manager [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] [ 1872.952916] env[62506]: DEBUG nova.compute.utils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1872.953919] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.508s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1872.955309] env[62506]: INFO nova.compute.claims [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1872.961336] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Build of instance e433b5a0-2742-4897-a7d9-46571bcc0f2c was re-scheduled: Binding failed for port 4acc1eca-d65e-4e91-a872-24d025379936, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1872.961881] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1872.962152] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquiring lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1872.962372] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Acquired lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1872.964635] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1872.976390] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1872.976390] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1872.976390] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1872.976390] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1872.976549] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1872.976549] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1872.976914] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1872.977254] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1872.977584] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1872.977905] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1872.978230] env[62506]: DEBUG nova.virt.hardware [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1872.979646] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-573e0831-5375-4baf-b794-156a30f902f6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1872.992137] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba64f2d3-63d4-4c01-b173-6d71430ad7f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1873.020516] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190797, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.144101} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1873.020797] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 1873.021704] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-797550a8-a087-4cb9-a012-1f67d0813087 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1873.044271] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Reconfiguring VM instance instance-00000010 to attach disk [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 1873.044631] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-ca0b9de4-f401-4647-aa18-c0202214f376 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1873.065791] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1873.065791] env[62506]: value = "task-2190798" [ 1873.065791] env[62506]: _type = "Task" [ 1873.065791] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1873.076509] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190798, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1873.153023] env[62506]: DEBUG nova.compute.manager [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Received event network-changed-db27c24c-5cdb-4c11-b330-073ad4c46df0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1873.153249] env[62506]: DEBUG nova.compute.manager [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Refreshing instance network info cache due to event network-changed-db27c24c-5cdb-4c11-b330-073ad4c46df0. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1873.153499] env[62506]: DEBUG oslo_concurrency.lockutils [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] Acquiring lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1873.153684] env[62506]: DEBUG oslo_concurrency.lockutils [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] Acquired lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1873.153854] env[62506]: DEBUG nova.network.neutron [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Refreshing network info cache for port db27c24c-5cdb-4c11-b330-073ad4c46df0 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1873.384908] env[62506]: ERROR nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1873.384908] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1873.384908] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1873.384908] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1873.384908] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1873.384908] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1873.384908] env[62506]: ERROR nova.compute.manager raise self.value [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1873.384908] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1873.384908] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1873.384908] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1873.385383] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1873.385383] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1873.385383] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1873.385383] env[62506]: ERROR nova.compute.manager [ 1873.385383] env[62506]: Traceback (most recent call last): [ 1873.385383] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1873.385383] env[62506]: listener.cb(fileno) [ 1873.385383] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1873.385383] env[62506]: result = function(*args, **kwargs) [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1873.385383] env[62506]: return func(*args, **kwargs) [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1873.385383] env[62506]: raise e [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1873.385383] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1873.385383] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1873.385383] env[62506]: with excutils.save_and_reraise_exception(): [ 1873.385383] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1873.385383] env[62506]: self.force_reraise() [ 1873.385383] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1873.385383] env[62506]: raise self.value [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1873.385383] env[62506]: updated_port = self._update_port( [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1873.385383] env[62506]: _ensure_no_port_binding_failure(port) [ 1873.385383] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1873.385383] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1873.386220] env[62506]: nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1873.386220] env[62506]: Removing descriptor: 19 [ 1873.386220] env[62506]: ERROR nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Traceback (most recent call last): [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] yield resources [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self.driver.spawn(context, instance, image_meta, [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1873.386220] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] vm_ref = self.build_virtual_machine(instance, [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] vif_infos = vmwarevif.get_vif_info(self._session, [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] for vif in network_info: [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return self._sync_wrapper(fn, *args, **kwargs) [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self.wait() [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self[:] = self._gt.wait() [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return self._exit_event.wait() [ 1873.386541] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] result = hub.switch() [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return self.greenlet.switch() [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] result = function(*args, **kwargs) [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return func(*args, **kwargs) [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] raise e [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] nwinfo = self.network_api.allocate_for_instance( [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1873.387055] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] created_port_ids = self._update_ports_for_instance( [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] with excutils.save_and_reraise_exception(): [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self.force_reraise() [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] raise self.value [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] updated_port = self._update_port( [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] _ensure_no_port_binding_failure(port) [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1873.387353] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] raise exception.PortBindingFailed(port_id=port['id']) [ 1873.387675] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1873.387675] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] [ 1873.387675] env[62506]: INFO nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Terminating instance [ 1873.487193] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1873.579028] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190798, 'name': ReconfigVM_Task, 'duration_secs': 0.278486} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1873.579028] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Reconfigured VM instance instance-00000010 to attach disk [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1873.579028] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-1e22fd6b-fac3-4252-8c46-5932d21a2ae0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1873.584572] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1873.584572] env[62506]: value = "task-2190799" [ 1873.584572] env[62506]: _type = "Task" [ 1873.584572] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1873.593128] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190799, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1873.603915] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1873.677887] env[62506]: DEBUG nova.network.neutron [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1873.837650] env[62506]: DEBUG nova.network.neutron [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1873.895943] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquiring lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1874.100146] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190799, 'name': Rename_Task, 'duration_secs': 0.132235} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1874.102602] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 1874.103047] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-d6e5b353-9a88-49b5-9539-8ffb2160d406 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1874.106529] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Releasing lock "refresh_cache-e433b5a0-2742-4897-a7d9-46571bcc0f2c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1874.106721] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1874.106888] env[62506]: DEBUG nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1874.107070] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1874.109803] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1874.109803] env[62506]: value = "task-2190800" [ 1874.109803] env[62506]: _type = "Task" [ 1874.109803] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1874.121023] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190800, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1874.131403] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1874.297440] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-327bf7ed-aa70-4864-81e1-ad04f39f27a6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1874.305289] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ea4e2bd-e29f-448a-88a0-8a244e2ba22c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1874.335892] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16dee292-a46b-47e3-9773-f6f16d6fc2de {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1874.338743] env[62506]: DEBUG oslo_concurrency.lockutils [req-f252d078-25eb-403d-a981-319773f1afb1 req-266d2530-a4cb-4ee1-847d-f0bf8060c764 service nova] Releasing lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1874.339177] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquired lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1874.339314] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1874.345325] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57fa9ad5-e07c-48fe-b79c-a66df45d1046 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1874.359448] env[62506]: DEBUG nova.compute.provider_tree [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1874.502055] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1874.502337] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1874.620879] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190800, 'name': PowerOnVM_Task} progress is 76%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1874.634657] env[62506]: DEBUG nova.network.neutron [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1874.865537] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1874.885523] env[62506]: ERROR nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [req-c4c3555c-f139-4eb7-afef-8541dc841992] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-c4c3555c-f139-4eb7-afef-8541dc841992"}]} [ 1874.906744] env[62506]: DEBUG nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 1874.924386] env[62506]: DEBUG nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 1874.924843] env[62506]: DEBUG nova.compute.provider_tree [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1874.940520] env[62506]: DEBUG nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 1874.962462] env[62506]: DEBUG nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 1874.967464] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1875.125112] env[62506]: DEBUG oslo_vmware.api [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190800, 'name': PowerOnVM_Task, 'duration_secs': 0.876299} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1875.125501] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 1875.125917] env[62506]: DEBUG nova.compute.manager [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1875.126754] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-776ec696-af24-423a-85e0-338298e08cf4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.140128] env[62506]: INFO nova.compute.manager [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] [instance: e433b5a0-2742-4897-a7d9-46571bcc0f2c] Took 1.03 seconds to deallocate network for instance. [ 1875.182156] env[62506]: DEBUG nova.compute.manager [req-d940f773-3aba-4179-a91c-55180e5a9cf2 req-4e1e76d0-582a-41e1-a60f-4d042835f48c service nova] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Received event network-vif-deleted-db27c24c-5cdb-4c11-b330-073ad4c46df0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1875.372360] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e34342ba-45c4-4ca1-a15c-d19239c3928b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.381058] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35ecbd56-9ff2-48eb-9610-43a96f2d048b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.411085] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-920a212f-114c-4099-b5e2-d930bbc8054e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.418981] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-254b9bff-5f5b-4ce1-97ed-4db1e8199e38 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.434492] env[62506]: DEBUG nova.compute.provider_tree [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1875.470215] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Releasing lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1875.470851] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1875.471095] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1875.471407] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4f963eef-10d5-44b2-8da0-a6b6716a1209 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.481308] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4a522f0-c0cc-45f8-b594-3c0ff00b822a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1875.506876] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6cdfeb65-c53f-4ba6-873c-88e593909878 could not be found. [ 1875.506876] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1875.506876] env[62506]: INFO nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1875.506876] env[62506]: DEBUG oslo.service.loopingcall [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1875.506876] env[62506]: DEBUG nova.compute.manager [-] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1875.506876] env[62506]: DEBUG nova.network.neutron [-] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1875.523216] env[62506]: DEBUG nova.network.neutron [-] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1875.661414] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1875.969865] env[62506]: DEBUG nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 53 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1875.970190] env[62506]: DEBUG nova.compute.provider_tree [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 53 to 54 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1875.970412] env[62506]: DEBUG nova.compute.provider_tree [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1876.026344] env[62506]: DEBUG nova.network.neutron [-] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1876.180506] env[62506]: INFO nova.scheduler.client.report [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Deleted allocations for instance e433b5a0-2742-4897-a7d9-46571bcc0f2c [ 1876.407394] env[62506]: INFO nova.compute.manager [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Rebuilding instance [ 1876.443042] env[62506]: DEBUG nova.compute.manager [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1876.443712] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1843a772-2026-4583-84b3-e5b0b569a0a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1876.475738] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.522s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1876.476258] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1876.478680] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.083s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1876.480249] env[62506]: INFO nova.compute.claims [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1876.529284] env[62506]: INFO nova.compute.manager [-] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Took 1.02 seconds to deallocate network for instance. [ 1876.531448] env[62506]: DEBUG nova.compute.claims [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1876.531626] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1876.688526] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a256353-7933-43e0-9f94-0aca69328133 tempest-FloatingIPsAssociationNegativeTestJSON-320585928 tempest-FloatingIPsAssociationNegativeTestJSON-320585928-project-member] Lock "e433b5a0-2742-4897-a7d9-46571bcc0f2c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 65.600s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1876.986409] env[62506]: DEBUG nova.compute.utils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1876.988460] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1876.988645] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1877.190931] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1877.241727] env[62506]: DEBUG nova.policy [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5624a4128dd942089b9d540e7117b700', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2469d590f72a4b59ba251dd6e5e95065', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1877.459282] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 1877.459617] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-0bdfc3e2-4c88-4d3c-863c-2d7a50ebd2ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1877.468757] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1877.468757] env[62506]: value = "task-2190801" [ 1877.468757] env[62506]: _type = "Task" [ 1877.468757] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1877.479136] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190801, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1877.492184] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1877.713929] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1877.900138] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3c77a7f-5e01-4099-beb8-30bd96c027d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1877.908027] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-323a5f0a-0f1e-458b-aade-69a9dd21cc06 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1877.943011] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-678aad16-b89d-4387-89fe-3a559f8276cc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1877.951296] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4efad0ea-454a-4d99-b374-1d2586ff4def {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1877.967138] env[62506]: DEBUG nova.compute.provider_tree [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1877.979019] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190801, 'name': PowerOffVM_Task, 'duration_secs': 0.209571} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1877.979373] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 1877.979616] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1877.980396] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4f595c9-e14f-476b-82f7-9c172065417b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1877.987418] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 1877.987872] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a03a0af1-5e61-418a-9529-3f6b49340b21 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1878.016183] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 1878.016479] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 1878.016742] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Deleting the datastore file [datastore2] af042cab-9132-4425-b017-133ec85afa0c {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1878.017058] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-b9468bad-30c9-418d-bf2a-a9bd2f8cab8a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1878.023838] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1878.023838] env[62506]: value = "task-2190803" [ 1878.023838] env[62506]: _type = "Task" [ 1878.023838] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1878.031981] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190803, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1878.036983] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Successfully created port: ed9e4982-0738-4d5a-bbcb-4ec0b9950baa {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1878.470640] env[62506]: DEBUG nova.scheduler.client.report [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1878.500443] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1878.532154] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1878.532448] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1878.532609] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1878.532795] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1878.532943] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1878.533185] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1878.533424] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1878.533588] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1878.534225] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1878.534411] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1878.534591] env[62506]: DEBUG nova.virt.hardware [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1878.535665] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5dcca0f-64a5-48fd-947a-1468069c1b73 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1878.542535] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190803, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.094389} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1878.543123] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1878.544232] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 1878.544442] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1878.553478] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5edec481-6adc-421f-a2cb-09ac5a2202d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1878.976934] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.498s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1878.977372] env[62506]: DEBUG nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1878.980389] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.879s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1878.981793] env[62506]: INFO nova.compute.claims [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1879.108113] env[62506]: DEBUG nova.compute.manager [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Received event network-changed-ed9e4982-0738-4d5a-bbcb-4ec0b9950baa {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1879.108364] env[62506]: DEBUG nova.compute.manager [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Refreshing instance network info cache due to event network-changed-ed9e4982-0738-4d5a-bbcb-4ec0b9950baa. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1879.108606] env[62506]: DEBUG oslo_concurrency.lockutils [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] Acquiring lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1879.108755] env[62506]: DEBUG oslo_concurrency.lockutils [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] Acquired lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1879.108972] env[62506]: DEBUG nova.network.neutron [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Refreshing network info cache for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1879.259561] env[62506]: ERROR nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1879.259561] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1879.259561] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1879.259561] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1879.259561] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1879.259561] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1879.259561] env[62506]: ERROR nova.compute.manager raise self.value [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1879.259561] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1879.259561] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1879.259561] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1879.260179] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1879.260179] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1879.260179] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1879.260179] env[62506]: ERROR nova.compute.manager [ 1879.260179] env[62506]: Traceback (most recent call last): [ 1879.260179] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1879.260179] env[62506]: listener.cb(fileno) [ 1879.260179] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1879.260179] env[62506]: result = function(*args, **kwargs) [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1879.260179] env[62506]: return func(*args, **kwargs) [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1879.260179] env[62506]: raise e [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1879.260179] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1879.260179] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1879.260179] env[62506]: with excutils.save_and_reraise_exception(): [ 1879.260179] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1879.260179] env[62506]: self.force_reraise() [ 1879.260179] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1879.260179] env[62506]: raise self.value [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1879.260179] env[62506]: updated_port = self._update_port( [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1879.260179] env[62506]: _ensure_no_port_binding_failure(port) [ 1879.260179] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1879.260179] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1879.260938] env[62506]: nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1879.260938] env[62506]: Removing descriptor: 19 [ 1879.260938] env[62506]: ERROR nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Traceback (most recent call last): [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] yield resources [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self.driver.spawn(context, instance, image_meta, [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1879.260938] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] vm_ref = self.build_virtual_machine(instance, [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] for vif in network_info: [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return self._sync_wrapper(fn, *args, **kwargs) [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self.wait() [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self[:] = self._gt.wait() [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return self._exit_event.wait() [ 1879.261285] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] result = hub.switch() [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return self.greenlet.switch() [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] result = function(*args, **kwargs) [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return func(*args, **kwargs) [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] raise e [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] nwinfo = self.network_api.allocate_for_instance( [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1879.261636] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] created_port_ids = self._update_ports_for_instance( [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] with excutils.save_and_reraise_exception(): [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self.force_reraise() [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] raise self.value [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] updated_port = self._update_port( [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] _ensure_no_port_binding_failure(port) [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1879.261994] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] raise exception.PortBindingFailed(port_id=port['id']) [ 1879.262363] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1879.262363] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] [ 1879.262363] env[62506]: INFO nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Terminating instance [ 1879.489373] env[62506]: DEBUG nova.compute.utils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1879.494584] env[62506]: DEBUG nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 1879.593240] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1879.593501] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1879.593658] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1879.593841] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1879.594146] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1879.594146] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1879.594366] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1879.594524] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1879.594695] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1879.594858] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1879.595041] env[62506]: DEBUG nova.virt.hardware [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1879.595897] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7a6130c-b8b9-42a2-b0f1-f55874d2964a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.605029] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d60f32a-397e-407a-b294-042d94452f6e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.620279] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1879.625955] env[62506]: DEBUG oslo.service.loopingcall [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1879.627529] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 1879.627529] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-176650c5-7348-4e1b-ba0a-2d0fc64d87c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.646721] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1879.646721] env[62506]: value = "task-2190804" [ 1879.646721] env[62506]: _type = "Task" [ 1879.646721] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1879.654654] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190804, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1879.655773] env[62506]: DEBUG nova.network.neutron [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1879.769040] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquiring lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1879.835118] env[62506]: DEBUG nova.network.neutron [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1879.995450] env[62506]: DEBUG nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1880.157063] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190804, 'name': CreateVM_Task, 'duration_secs': 0.240421} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1880.157250] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 1880.157694] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1880.157867] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1880.158314] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1880.158489] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2c1f8f73-620a-4878-b39a-c06f490ec803 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.163440] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1880.163440] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]521474cf-3115-7585-df58-81f280b11b00" [ 1880.163440] env[62506]: _type = "Task" [ 1880.163440] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1880.174605] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]521474cf-3115-7585-df58-81f280b11b00, 'name': SearchDatastore_Task, 'duration_secs': 0.009147} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1880.174881] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1880.175549] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1880.175549] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1880.175549] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1880.175729] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1880.175941] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e644f7ab-48d9-4855-9e97-f8482e9d1e20 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.183034] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1880.183226] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 1880.183927] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-85fd3c45-0998-463a-bf84-e37b22892885 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.188884] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1880.188884] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5209d968-1eb1-d708-eae7-d461823e5961" [ 1880.188884] env[62506]: _type = "Task" [ 1880.188884] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1880.198745] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5209d968-1eb1-d708-eae7-d461823e5961, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1880.332789] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecfc3314-2c9e-4087-b7c4-873ce5c0199d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.340582] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ea022b7-95b4-4477-a4e1-bcd4f1787fd4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.344620] env[62506]: DEBUG oslo_concurrency.lockutils [req-c852b1dc-58f3-401f-9cf4-222242002586 req-c79af7e8-3772-4530-9636-dbf4619fe867 service nova] Releasing lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1880.344983] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquired lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1880.345206] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1880.377286] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a66b6d1-7e75-4ca4-8741-41b63583e1c6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.386587] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8958dc3-5621-4582-ac91-0eee0295745f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.404355] env[62506]: DEBUG nova.compute.provider_tree [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1880.702113] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5209d968-1eb1-d708-eae7-d461823e5961, 'name': SearchDatastore_Task, 'duration_secs': 0.009233} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1880.702883] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-94253ace-11b0-4d85-aef4-ff0e6089e6c9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.707913] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1880.707913] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52344b8f-0949-322b-941d-3745f05cd3e9" [ 1880.707913] env[62506]: _type = "Task" [ 1880.707913] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1880.715503] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52344b8f-0949-322b-941d-3745f05cd3e9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1880.876165] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1880.963517] env[62506]: DEBUG nova.scheduler.client.report [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 54 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1880.963786] env[62506]: DEBUG nova.compute.provider_tree [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 54 to 55 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1880.963971] env[62506]: DEBUG nova.compute.provider_tree [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1881.007201] env[62506]: DEBUG nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1881.032589] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1881.032854] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1881.033025] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1881.033221] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1881.033368] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1881.033517] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1881.033726] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1881.033887] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1881.034068] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1881.034241] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1881.034416] env[62506]: DEBUG nova.virt.hardware [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1881.035271] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b01aebcb-b61d-49f6-81b3-afbb0052db51 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.043240] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dab3ff1-8040-4b77-9026-2529e57fe4cd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.057030] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1881.062785] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Creating folder: Project (bc689ee2e99d483984bf02bd8bfefe38). Parent ref: group-v446953. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1881.063088] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-259aa682-b54d-4b3a-9d37-14029812059a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.065772] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1881.073507] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Created folder: Project (bc689ee2e99d483984bf02bd8bfefe38) in parent group-v446953. [ 1881.073660] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Creating folder: Instances. Parent ref: group-v446964. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1881.073900] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b4ac12da-8142-427a-89d8-6283343105ea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.082530] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Created folder: Instances in parent group-v446964. [ 1881.082777] env[62506]: DEBUG oslo.service.loopingcall [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1881.082961] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 1881.083175] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-efd91a2e-4353-4b65-8a2d-1b5b72d98e57 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.101143] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1881.101143] env[62506]: value = "task-2190807" [ 1881.101143] env[62506]: _type = "Task" [ 1881.101143] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1881.108966] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190807, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1881.168989] env[62506]: DEBUG nova.compute.manager [req-1a30f48c-0a81-4734-937e-c96ce30263b9 req-6d60882e-b110-41e3-9efb-2094758a0aec service nova] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Received event network-vif-deleted-ed9e4982-0738-4d5a-bbcb-4ec0b9950baa {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1881.218250] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52344b8f-0949-322b-941d-3745f05cd3e9, 'name': SearchDatastore_Task, 'duration_secs': 0.01803} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1881.218920] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1881.219217] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 1881.219817] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-70dea485-0473-43ea-a4f2-944af48dd60b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.227486] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1881.227486] env[62506]: value = "task-2190808" [ 1881.227486] env[62506]: _type = "Task" [ 1881.227486] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1881.236693] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190808, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1881.469634] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.488s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1881.469634] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1881.475291] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.453s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1881.569300] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Releasing lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1881.569782] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1881.570037] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1881.570356] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7367ca84-9986-48f9-a0e1-3a600c5aa259 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.579973] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-942c7f84-2567-4bb9-9d65-2b494df7ee24 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.604579] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 249e5265-86b3-4ba8-96b9-d21ea0a5599d could not be found. [ 1881.604829] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1881.605040] env[62506]: INFO nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1881.605295] env[62506]: DEBUG oslo.service.loopingcall [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1881.608750] env[62506]: DEBUG nova.compute.manager [-] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1881.608859] env[62506]: DEBUG nova.network.neutron [-] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1881.617302] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190807, 'name': CreateVM_Task, 'duration_secs': 0.379745} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1881.617495] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 1881.617960] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1881.618172] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1881.618519] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1881.618831] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-58278bf3-22dd-4678-a122-fe587b8bcd8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.624145] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1881.624145] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52e78926-5760-ffbd-64b2-97400128c249" [ 1881.624145] env[62506]: _type = "Task" [ 1881.624145] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1881.629576] env[62506]: DEBUG nova.network.neutron [-] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1881.635370] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52e78926-5760-ffbd-64b2-97400128c249, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1881.743128] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190808, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.503852} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1881.743578] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 1881.743825] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 1881.743966] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-64a460e3-59bd-4b41-a722-e74d48d54928 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1881.750092] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1881.750092] env[62506]: value = "task-2190809" [ 1881.750092] env[62506]: _type = "Task" [ 1881.750092] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1881.758199] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190809, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1881.985084] env[62506]: DEBUG nova.compute.utils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1881.991606] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1881.992394] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1882.090460] env[62506]: DEBUG nova.policy [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '49d4e60cbc414fdc9c10a181b098a9a1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '793b228371974c71a10877d3b9f365f0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1882.138016] env[62506]: DEBUG nova.network.neutron [-] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1882.139327] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52e78926-5760-ffbd-64b2-97400128c249, 'name': SearchDatastore_Task, 'duration_secs': 0.048008} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1882.142112] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1882.144833] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1882.144833] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1882.144833] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1882.144833] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1882.144833] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e1b9f2fd-1d8e-44d3-be5c-44fcd583e2d2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.154293] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1882.154500] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 1882.155487] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4d7a7014-5d2b-4c85-8818-4b7fd4831a58 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.163050] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1882.163050] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]527aa987-89ea-8907-864d-17ff83bb83ee" [ 1882.163050] env[62506]: _type = "Task" [ 1882.163050] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1882.171888] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]527aa987-89ea-8907-864d-17ff83bb83ee, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1882.260806] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190809, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.06289} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1882.264488] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 1882.265872] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e83a0770-208c-4d04-a704-f5c46742534b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.292830] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Reconfiguring VM instance instance-00000010 to attach disk [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 1882.297210] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-0e14557a-96b7-443b-8df1-3a15c9217d3a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.320859] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1882.320859] env[62506]: value = "task-2190810" [ 1882.320859] env[62506]: _type = "Task" [ 1882.320859] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1882.330328] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190810, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1882.491473] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1882.498140] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb7b5871-a8d6-4531-843c-bad78419a54d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.506686] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3050576f-0282-45a5-a718-69c98cfd9008 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.540521] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8bbd5a9-6ebd-4166-8fea-d4ad77128ff9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.548889] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19fce97a-4097-4796-a372-14af65036687 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.565204] env[62506]: DEBUG nova.compute.provider_tree [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1882.643361] env[62506]: INFO nova.compute.manager [-] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Took 1.03 seconds to deallocate network for instance. [ 1882.646138] env[62506]: DEBUG nova.compute.claims [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1882.646345] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1882.671161] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Successfully created port: e58bb70a-f560-4376-92b6-ae80163c2364 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1882.680265] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]527aa987-89ea-8907-864d-17ff83bb83ee, 'name': SearchDatastore_Task, 'duration_secs': 0.014017} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1882.680265] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-be1c8584-d9dd-4516-acf8-ac75a90ed844 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.685367] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1882.685367] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52f5b730-3504-0f85-7abc-9cfbf4a52a11" [ 1882.685367] env[62506]: _type = "Task" [ 1882.685367] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1882.692989] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52f5b730-3504-0f85-7abc-9cfbf4a52a11, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1882.830928] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190810, 'name': ReconfigVM_Task, 'duration_secs': 0.26938} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1882.831335] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Reconfigured VM instance instance-00000010 to attach disk [datastore2] af042cab-9132-4425-b017-133ec85afa0c/af042cab-9132-4425-b017-133ec85afa0c.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1882.831834] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-fcadcff1-12de-43a5-af4d-4170a301be03 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1882.839153] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1882.839153] env[62506]: value = "task-2190811" [ 1882.839153] env[62506]: _type = "Task" [ 1882.839153] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1882.847593] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190811, 'name': Rename_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1883.102831] env[62506]: DEBUG nova.scheduler.client.report [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 55 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 1883.103138] env[62506]: DEBUG nova.compute.provider_tree [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 55 to 56 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1883.103693] env[62506]: DEBUG nova.compute.provider_tree [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1883.195836] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52f5b730-3504-0f85-7abc-9cfbf4a52a11, 'name': SearchDatastore_Task, 'duration_secs': 0.009288} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1883.196137] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1883.196399] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] b4bcab92-0007-4952-8d74-c470049edd71/b4bcab92-0007-4952-8d74-c470049edd71.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 1883.196653] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-64c901b0-85db-4dd2-982f-7eb0aa96bbb7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1883.203153] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1883.203153] env[62506]: value = "task-2190812" [ 1883.203153] env[62506]: _type = "Task" [ 1883.203153] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1883.211921] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190812, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1883.355089] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190811, 'name': Rename_Task, 'duration_secs': 0.132349} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1883.355438] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 1883.355693] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-58835c8e-e4be-433c-b8a4-a7606c58d55e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1883.362609] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Waiting for the task: (returnval){ [ 1883.362609] env[62506]: value = "task-2190813" [ 1883.362609] env[62506]: _type = "Task" [ 1883.362609] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1883.373483] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190813, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1883.501344] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1883.532218] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1883.532218] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1883.532218] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1883.532558] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1883.532558] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1883.532558] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1883.532958] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1883.533208] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1883.533458] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1883.533699] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1883.533936] env[62506]: DEBUG nova.virt.hardware [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1883.534843] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-790736cb-4fe8-4384-ba72-c281aaf9aad0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1883.543919] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08643ee5-9c2e-4195-9b20-4a88a4d8ee98 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1883.608975] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.135s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1883.609500] env[62506]: ERROR nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Traceback (most recent call last): [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self.driver.spawn(context, instance, image_meta, [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] vm_ref = self.build_virtual_machine(instance, [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] vif_infos = vmwarevif.get_vif_info(self._session, [ 1883.609500] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] for vif in network_info: [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return self._sync_wrapper(fn, *args, **kwargs) [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self.wait() [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self[:] = self._gt.wait() [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return self._exit_event.wait() [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] result = hub.switch() [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1883.609807] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return self.greenlet.switch() [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] result = function(*args, **kwargs) [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] return func(*args, **kwargs) [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] raise e [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] nwinfo = self.network_api.allocate_for_instance( [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] created_port_ids = self._update_ports_for_instance( [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] with excutils.save_and_reraise_exception(): [ 1883.610201] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] self.force_reraise() [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] raise self.value [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] updated_port = self._update_port( [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] _ensure_no_port_binding_failure(port) [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] raise exception.PortBindingFailed(port_id=port['id']) [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] nova.exception.PortBindingFailed: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. [ 1883.610503] env[62506]: ERROR nova.compute.manager [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] [ 1883.610762] env[62506]: DEBUG nova.compute.utils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1883.612225] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.559s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1883.616314] env[62506]: INFO nova.compute.claims [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1883.620357] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Build of instance 386935e4-c2b7-4207-9670-cd532ff12f70 was re-scheduled: Binding failed for port cd070db2-3b79-4455-bcb8-7cdca2b63e18, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1883.620357] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1883.621126] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquiring lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1883.621542] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Acquired lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1883.621802] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1883.716156] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190812, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.507455} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1883.717425] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] b4bcab92-0007-4952-8d74-c470049edd71/b4bcab92-0007-4952-8d74-c470049edd71.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 1883.717703] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 1883.718353] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-8084b4c9-de90-4627-8581-8e8c9b080cac {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1883.725502] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1883.725502] env[62506]: value = "task-2190814" [ 1883.725502] env[62506]: _type = "Task" [ 1883.725502] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1883.737278] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190814, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1883.875956] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190813, 'name': PowerOnVM_Task} progress is 87%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1883.994723] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "8f808943-6107-49d9-966f-c4e6ae0cdac4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1883.995245] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "8f808943-6107-49d9-966f-c4e6ae0cdac4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1884.141967] env[62506]: DEBUG nova.compute.manager [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Received event network-changed-e58bb70a-f560-4376-92b6-ae80163c2364 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1884.143206] env[62506]: DEBUG nova.compute.manager [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Refreshing instance network info cache due to event network-changed-e58bb70a-f560-4376-92b6-ae80163c2364. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1884.143206] env[62506]: DEBUG oslo_concurrency.lockutils [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] Acquiring lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1884.143206] env[62506]: DEBUG oslo_concurrency.lockutils [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] Acquired lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1884.143381] env[62506]: DEBUG nova.network.neutron [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Refreshing network info cache for port e58bb70a-f560-4376-92b6-ae80163c2364 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1884.153880] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1884.235160] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190814, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.133956} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1884.235441] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 1884.236252] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32334bfb-f724-4dd2-947d-43d0d7c6209e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1884.255842] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Reconfiguring VM instance instance-00000015 to attach disk [datastore2] b4bcab92-0007-4952-8d74-c470049edd71/b4bcab92-0007-4952-8d74-c470049edd71.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 1884.256117] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-26130ddb-7c5f-4c6e-a8de-954d07617fca {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1884.275329] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1884.279245] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1884.279245] env[62506]: value = "task-2190815" [ 1884.279245] env[62506]: _type = "Task" [ 1884.279245] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1884.290870] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190815, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1884.372650] env[62506]: DEBUG oslo_vmware.api [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Task: {'id': task-2190813, 'name': PowerOnVM_Task, 'duration_secs': 0.70728} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1884.373750] env[62506]: ERROR nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1884.373750] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1884.373750] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1884.373750] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1884.373750] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1884.373750] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1884.373750] env[62506]: ERROR nova.compute.manager raise self.value [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1884.373750] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1884.373750] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1884.373750] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1884.374533] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1884.374533] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1884.374533] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1884.374533] env[62506]: ERROR nova.compute.manager [ 1884.374533] env[62506]: Traceback (most recent call last): [ 1884.374533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1884.374533] env[62506]: listener.cb(fileno) [ 1884.374533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1884.374533] env[62506]: result = function(*args, **kwargs) [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1884.374533] env[62506]: return func(*args, **kwargs) [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1884.374533] env[62506]: raise e [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1884.374533] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1884.374533] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1884.374533] env[62506]: with excutils.save_and_reraise_exception(): [ 1884.374533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1884.374533] env[62506]: self.force_reraise() [ 1884.374533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1884.374533] env[62506]: raise self.value [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1884.374533] env[62506]: updated_port = self._update_port( [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1884.374533] env[62506]: _ensure_no_port_binding_failure(port) [ 1884.374533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1884.374533] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1884.375330] env[62506]: nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1884.375330] env[62506]: Removing descriptor: 19 [ 1884.375330] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 1884.375330] env[62506]: DEBUG nova.compute.manager [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1884.375330] env[62506]: ERROR nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1884.375330] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Traceback (most recent call last): [ 1884.375330] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1884.375330] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] yield resources [ 1884.375330] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1884.375330] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self.driver.spawn(context, instance, image_meta, [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] vm_ref = self.build_virtual_machine(instance, [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] vif_infos = vmwarevif.get_vif_info(self._session, [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] for vif in network_info: [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return self._sync_wrapper(fn, *args, **kwargs) [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self.wait() [ 1884.375580] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self[:] = self._gt.wait() [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return self._exit_event.wait() [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] result = hub.switch() [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return self.greenlet.switch() [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] result = function(*args, **kwargs) [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return func(*args, **kwargs) [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1884.375993] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] raise e [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] nwinfo = self.network_api.allocate_for_instance( [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] created_port_ids = self._update_ports_for_instance( [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] with excutils.save_and_reraise_exception(): [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self.force_reraise() [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] raise self.value [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] updated_port = self._update_port( [ 1884.376357] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1884.376676] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] _ensure_no_port_binding_failure(port) [ 1884.376676] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1884.376676] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] raise exception.PortBindingFailed(port_id=port['id']) [ 1884.376676] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1884.376676] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] [ 1884.376676] env[62506]: INFO nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Terminating instance [ 1884.377372] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22911202-883d-4577-9a3c-fc16a1236ddd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1884.667798] env[62506]: DEBUG nova.network.neutron [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1884.778816] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Releasing lock "refresh_cache-386935e4-c2b7-4207-9670-cd532ff12f70" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1884.779308] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1884.779308] env[62506]: DEBUG nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1884.779403] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1884.781739] env[62506]: DEBUG nova.network.neutron [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1884.797698] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190815, 'name': ReconfigVM_Task, 'duration_secs': 0.288204} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1884.798011] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Reconfigured VM instance instance-00000015 to attach disk [datastore2] b4bcab92-0007-4952-8d74-c470049edd71/b4bcab92-0007-4952-8d74-c470049edd71.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1884.798835] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-004a90bd-b963-441a-9739-d8e858b8fa07 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1884.810140] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1884.810140] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1884.810140] env[62506]: value = "task-2190816" [ 1884.810140] env[62506]: _type = "Task" [ 1884.810140] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1884.824783] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190816, 'name': Rename_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1884.888664] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquiring lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1884.898153] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1885.054889] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7aea5a85-d6e1-4337-851e-891d8630365e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1885.062405] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3c548ee-d710-48eb-9947-64396a881de9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1885.093506] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c5edfee-182c-4fd6-a431-712947c7a5f4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1885.101247] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37b56e9e-6870-41ce-9ec2-fd237889b32b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1885.114362] env[62506]: DEBUG nova.compute.provider_tree [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1885.287025] env[62506]: DEBUG oslo_concurrency.lockutils [req-003ce95c-8f01-49fe-8ef3-033e7b772b05 req-f40b2a5d-bcd6-4924-bc93-5fd0f6b31d99 service nova] Releasing lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1885.287305] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquired lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1885.287364] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1885.315078] env[62506]: DEBUG nova.network.neutron [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1885.321362] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190816, 'name': Rename_Task, 'duration_secs': 0.206543} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1885.321608] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 1885.321838] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-1fde53d6-a5b0-4f54-831f-e9f5afa8f6dd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1885.328793] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1885.328793] env[62506]: value = "task-2190817" [ 1885.328793] env[62506]: _type = "Task" [ 1885.328793] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1885.336447] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190817, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1885.617978] env[62506]: DEBUG nova.scheduler.client.report [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1885.702059] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "af042cab-9132-4425-b017-133ec85afa0c" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1885.702158] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "af042cab-9132-4425-b017-133ec85afa0c" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1885.702379] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "af042cab-9132-4425-b017-133ec85afa0c-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1885.702568] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "af042cab-9132-4425-b017-133ec85afa0c-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1885.702735] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "af042cab-9132-4425-b017-133ec85afa0c-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1885.704809] env[62506]: INFO nova.compute.manager [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Terminating instance [ 1885.817692] env[62506]: INFO nova.compute.manager [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] [instance: 386935e4-c2b7-4207-9670-cd532ff12f70] Took 1.04 seconds to deallocate network for instance. [ 1885.821271] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1885.839082] env[62506]: DEBUG oslo_vmware.api [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190817, 'name': PowerOnVM_Task, 'duration_secs': 0.389819} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1885.839082] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 1885.839209] env[62506]: INFO nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Took 4.83 seconds to spawn the instance on the hypervisor. [ 1885.839337] env[62506]: DEBUG nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1885.840130] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbfb38f4-eac0-4065-be2c-d4ec1e78571d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1885.924925] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1886.123570] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.511s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1886.124119] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1886.126813] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.474s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1886.128284] env[62506]: INFO nova.compute.claims [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1886.187030] env[62506]: DEBUG nova.compute.manager [None req-9e068d2e-87ff-4375-ac16-83e85e5219d1 tempest-ServerDiagnosticsV248Test-1295293030 tempest-ServerDiagnosticsV248Test-1295293030-project-admin] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1886.189106] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-040e7ee8-a8c4-4931-a266-f9b9dd34adc1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1886.196157] env[62506]: INFO nova.compute.manager [None req-9e068d2e-87ff-4375-ac16-83e85e5219d1 tempest-ServerDiagnosticsV248Test-1295293030 tempest-ServerDiagnosticsV248Test-1295293030-project-admin] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Retrieving diagnostics [ 1886.197121] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4d5f5bc-e543-4f41-b644-644aca2b25ce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1886.228671] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "refresh_cache-af042cab-9132-4425-b017-133ec85afa0c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1886.228847] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquired lock "refresh_cache-af042cab-9132-4425-b017-133ec85afa0c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1886.229032] env[62506]: DEBUG nova.network.neutron [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1886.232315] env[62506]: DEBUG nova.compute.manager [req-2bfd086b-386f-49bc-a7c0-432548a8b1a5 req-3421a289-7db0-4ed4-9805-a98d2a2d9a2d service nova] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Received event network-vif-deleted-e58bb70a-f560-4376-92b6-ae80163c2364 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1886.356938] env[62506]: INFO nova.compute.manager [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Took 29.98 seconds to build instance. [ 1886.427874] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Releasing lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1886.428357] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1886.428560] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1886.428858] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-709543ca-d440-493c-8732-45d8ec0e9a18 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1886.437774] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fd50e0f-0535-4189-9d14-4217e333a64d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1886.460456] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a7ed3eba-aced-4ff9-ba98-e29d44ef5c54 could not be found. [ 1886.460687] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1886.460882] env[62506]: INFO nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1886.461235] env[62506]: DEBUG oslo.service.loopingcall [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1886.461475] env[62506]: DEBUG nova.compute.manager [-] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1886.461572] env[62506]: DEBUG nova.network.neutron [-] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1886.482537] env[62506]: DEBUG nova.network.neutron [-] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1886.636286] env[62506]: DEBUG nova.compute.utils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1886.637681] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1886.637889] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1886.697740] env[62506]: DEBUG nova.policy [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8615160e68af4fcf9e9cd0d9f97a0b3e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0e4a3d4707b74087b11ee299cd02976c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1886.763257] env[62506]: DEBUG nova.network.neutron [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1886.859079] env[62506]: INFO nova.scheduler.client.report [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Deleted allocations for instance 386935e4-c2b7-4207-9670-cd532ff12f70 [ 1886.865875] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b833dc31-a182-4d23-8d93-f6c9986bf639 tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "b4bcab92-0007-4952-8d74-c470049edd71" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 70.878s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1886.879282] env[62506]: DEBUG nova.network.neutron [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1886.985519] env[62506]: DEBUG nova.network.neutron [-] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1887.144351] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1887.245451] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Successfully created port: cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1887.368814] env[62506]: DEBUG oslo_concurrency.lockutils [None req-aa1ba225-4d04-4162-98a6-6f180b2e44fd tempest-ServerRescueTestJSON-1793833045 tempest-ServerRescueTestJSON-1793833045-project-member] Lock "386935e4-c2b7-4207-9670-cd532ff12f70" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 75.489s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1887.369097] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1887.382710] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Releasing lock "refresh_cache-af042cab-9132-4425-b017-133ec85afa0c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1887.383099] env[62506]: DEBUG nova.compute.manager [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1887.383290] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1887.384357] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a71776bc-5f54-4ba5-8584-cdf342ac5c07 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.393416] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 1887.395870] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-d05980f2-db19-47b3-b146-ea38a3a4d9d5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.406313] env[62506]: DEBUG oslo_vmware.api [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1887.406313] env[62506]: value = "task-2190818" [ 1887.406313] env[62506]: _type = "Task" [ 1887.406313] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1887.427318] env[62506]: DEBUG oslo_vmware.api [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190818, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1887.490093] env[62506]: INFO nova.compute.manager [-] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Took 1.03 seconds to deallocate network for instance. [ 1887.492690] env[62506]: DEBUG nova.compute.claims [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1887.492868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1887.536956] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-793d37a7-aaf8-4dc7-9269-ba062b92de7b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.544986] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae10bdcb-cf1e-4c47-9b7b-253a0de5ac93 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.575478] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44445358-475d-4c88-b43b-18e6bc72ab50 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.583011] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a158231-c16e-4d32-b7f3-3aa393392376 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.598870] env[62506]: DEBUG nova.compute.provider_tree [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1887.878895] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1887.901070] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1887.917432] env[62506]: DEBUG oslo_vmware.api [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190818, 'name': PowerOffVM_Task, 'duration_secs': 0.188465} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1887.917716] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 1887.917884] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 1887.918197] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0436f282-d570-4ab5-ba2d-7894cb64a606 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.943466] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 1887.946032] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 1887.946032] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Deleting the datastore file [datastore2] af042cab-9132-4425-b017-133ec85afa0c {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1887.946032] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6d4279a7-f8bc-46d8-bfb1-5ed36b08cebb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1887.951519] env[62506]: DEBUG oslo_vmware.api [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for the task: (returnval){ [ 1887.951519] env[62506]: value = "task-2190820" [ 1887.951519] env[62506]: _type = "Task" [ 1887.951519] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1887.964724] env[62506]: DEBUG oslo_vmware.api [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190820, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1888.098975] env[62506]: DEBUG nova.scheduler.client.report [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1888.154683] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1888.180872] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1888.181149] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1888.181294] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1888.181481] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1888.181626] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1888.181773] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1888.181983] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1888.182157] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1888.182344] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1888.182528] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1888.182707] env[62506]: DEBUG nova.virt.hardware [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1888.183601] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4371e7c3-fcfb-41a2-b9f4-f5fb1be51d09 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1888.191923] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24862e25-17a3-47be-908e-91b0b211278a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1888.296090] env[62506]: DEBUG nova.compute.manager [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Received event network-changed-cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1888.296090] env[62506]: DEBUG nova.compute.manager [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Refreshing instance network info cache due to event network-changed-cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1888.296090] env[62506]: DEBUG oslo_concurrency.lockutils [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] Acquiring lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1888.296090] env[62506]: DEBUG oslo_concurrency.lockutils [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] Acquired lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1888.296090] env[62506]: DEBUG nova.network.neutron [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Refreshing network info cache for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1888.406313] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1888.461387] env[62506]: DEBUG oslo_vmware.api [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Task: {'id': task-2190820, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.189494} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1888.461698] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1888.461885] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 1888.462075] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1888.462253] env[62506]: INFO nova.compute.manager [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Took 1.08 seconds to destroy the instance on the hypervisor. [ 1888.462495] env[62506]: DEBUG oslo.service.loopingcall [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1888.462686] env[62506]: DEBUG nova.compute.manager [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1888.462784] env[62506]: DEBUG nova.network.neutron [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1888.499948] env[62506]: DEBUG nova.network.neutron [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1888.584115] env[62506]: ERROR nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1888.584115] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1888.584115] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1888.584115] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1888.584115] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1888.584115] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1888.584115] env[62506]: ERROR nova.compute.manager raise self.value [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1888.584115] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1888.584115] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1888.584115] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1888.584579] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1888.584579] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1888.584579] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1888.584579] env[62506]: ERROR nova.compute.manager [ 1888.584579] env[62506]: Traceback (most recent call last): [ 1888.584579] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1888.584579] env[62506]: listener.cb(fileno) [ 1888.584579] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1888.584579] env[62506]: result = function(*args, **kwargs) [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1888.584579] env[62506]: return func(*args, **kwargs) [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1888.584579] env[62506]: raise e [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1888.584579] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1888.584579] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1888.584579] env[62506]: with excutils.save_and_reraise_exception(): [ 1888.584579] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1888.584579] env[62506]: self.force_reraise() [ 1888.584579] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1888.584579] env[62506]: raise self.value [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1888.584579] env[62506]: updated_port = self._update_port( [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1888.584579] env[62506]: _ensure_no_port_binding_failure(port) [ 1888.584579] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1888.584579] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1888.585240] env[62506]: nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1888.585240] env[62506]: Removing descriptor: 15 [ 1888.585240] env[62506]: ERROR nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Traceback (most recent call last): [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] yield resources [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self.driver.spawn(context, instance, image_meta, [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1888.585240] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] vm_ref = self.build_virtual_machine(instance, [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] vif_infos = vmwarevif.get_vif_info(self._session, [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] for vif in network_info: [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return self._sync_wrapper(fn, *args, **kwargs) [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self.wait() [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self[:] = self._gt.wait() [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return self._exit_event.wait() [ 1888.585517] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] result = hub.switch() [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return self.greenlet.switch() [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] result = function(*args, **kwargs) [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return func(*args, **kwargs) [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] raise e [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] nwinfo = self.network_api.allocate_for_instance( [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1888.585819] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] created_port_ids = self._update_ports_for_instance( [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] with excutils.save_and_reraise_exception(): [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self.force_reraise() [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] raise self.value [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] updated_port = self._update_port( [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] _ensure_no_port_binding_failure(port) [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1888.586143] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] raise exception.PortBindingFailed(port_id=port['id']) [ 1888.586420] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1888.586420] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] [ 1888.586420] env[62506]: INFO nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Terminating instance [ 1888.609414] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.482s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1888.609929] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1888.612626] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 19.495s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1888.612826] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1888.612984] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1888.613645] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.106s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1888.617469] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62373f4a-ab81-412d-9bdf-c5d20c07afc4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1888.627272] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-951871e0-53c7-41fa-8c9c-4adc22143745 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1888.644504] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8890f129-5800-4976-8386-9ef9908ff6eb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1888.652222] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33b682f5-af58-4e35-be62-88d8a31ce25c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1888.690750] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181225MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1888.690916] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1888.822407] env[62506]: DEBUG nova.network.neutron [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1888.971076] env[62506]: DEBUG nova.network.neutron [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1889.003022] env[62506]: DEBUG nova.network.neutron [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1889.093277] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1889.118818] env[62506]: DEBUG nova.compute.utils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1889.120763] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1889.120763] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1889.169363] env[62506]: DEBUG nova.policy [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'eb1d4a713d3f4dceaf2088278120924c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '073cbe62e4fb43ae80d2a6390785c14d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1889.468129] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70282c26-5b18-4a1f-a9d8-16d6b2e87468 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1889.476270] env[62506]: DEBUG oslo_concurrency.lockutils [req-3d5a1c3b-63c1-41b6-ba92-6b8aa956539c req-8b00f191-ef6b-4caa-b3a8-8021f84f7f6f service nova] Releasing lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1889.476729] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1889.476905] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1889.478749] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6e23785-db5f-4a83-b1ce-403a70ebe9cd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1889.514574] env[62506]: INFO nova.compute.manager [-] [instance: af042cab-9132-4425-b017-133ec85afa0c] Took 1.05 seconds to deallocate network for instance. [ 1889.519066] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-128f2931-b719-4ec4-8f26-8afff35286ab {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1889.528514] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a70f85f-2eda-43d7-8ac2-73a47353a059 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1889.543133] env[62506]: DEBUG nova.compute.provider_tree [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1889.623866] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1889.717838] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Successfully created port: 1359683b-7547-41e9-8cc3-54926ba26963 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1890.014801] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1890.024385] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1890.050617] env[62506]: DEBUG nova.scheduler.client.report [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1890.207975] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1890.325929] env[62506]: DEBUG nova.compute.manager [req-814d9971-626b-4543-ae8b-9d2807b0fd7f req-40f11b00-d3a2-4e09-a09a-be38c931f0ce service nova] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Received event network-vif-deleted-cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1890.556415] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.943s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1890.557077] env[62506]: ERROR nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Traceback (most recent call last): [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self.driver.spawn(context, instance, image_meta, [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] vm_ref = self.build_virtual_machine(instance, [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] vif_infos = vmwarevif.get_vif_info(self._session, [ 1890.557077] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] for vif in network_info: [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return self._sync_wrapper(fn, *args, **kwargs) [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self.wait() [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self[:] = self._gt.wait() [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return self._exit_event.wait() [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] result = hub.switch() [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1890.557395] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return self.greenlet.switch() [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] result = function(*args, **kwargs) [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] return func(*args, **kwargs) [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] raise e [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] nwinfo = self.network_api.allocate_for_instance( [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] created_port_ids = self._update_ports_for_instance( [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] with excutils.save_and_reraise_exception(): [ 1890.557696] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] self.force_reraise() [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] raise self.value [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] updated_port = self._update_port( [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] _ensure_no_port_binding_failure(port) [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] raise exception.PortBindingFailed(port_id=port['id']) [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] nova.exception.PortBindingFailed: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. [ 1890.558033] env[62506]: ERROR nova.compute.manager [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] [ 1890.558296] env[62506]: DEBUG nova.compute.utils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1890.559403] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.441s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1890.560884] env[62506]: INFO nova.compute.claims [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1890.567017] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Build of instance 88322331-7cb9-46a9-8e8c-82f0198c90d8 was re-scheduled: Binding failed for port 285501f8-43f4-493c-85d9-9c87ba6dd1f8, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1890.567017] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1890.567017] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquiring lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1890.567017] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Acquired lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1890.567224] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1890.634028] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1890.672838] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1890.672838] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1890.672838] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1890.673049] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1890.673049] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1890.673113] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1890.673352] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1890.673521] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1890.673686] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1890.673853] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1890.674040] env[62506]: DEBUG nova.virt.hardware [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1890.674894] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12cfad0c-7bc5-4657-9dc8-6fafbac8d5c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1890.683645] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f5ecf38-758c-42f3-af89-3120d7f690d0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1890.710988] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1890.712903] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1890.715041] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1890.715041] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-201100e1-505f-42d4-9f61-216267ab93c4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1890.722996] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3262b20-76b0-4b84-b56c-16adaf4fe67c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1890.748865] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8fd213dd-d597-48c0-9387-19b4f5dd72ff could not be found. [ 1890.749127] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1890.749320] env[62506]: INFO nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1890.749562] env[62506]: DEBUG oslo.service.loopingcall [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1890.749793] env[62506]: DEBUG nova.compute.manager [-] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1890.749888] env[62506]: DEBUG nova.network.neutron [-] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1890.770446] env[62506]: DEBUG nova.network.neutron [-] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1890.994370] env[62506]: ERROR nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1890.994370] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1890.994370] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1890.994370] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1890.994370] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1890.994370] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1890.994370] env[62506]: ERROR nova.compute.manager raise self.value [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1890.994370] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1890.994370] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1890.994370] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1890.994823] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1890.994823] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1890.994823] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1890.994823] env[62506]: ERROR nova.compute.manager [ 1890.994823] env[62506]: Traceback (most recent call last): [ 1890.994823] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1890.994823] env[62506]: listener.cb(fileno) [ 1890.994823] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1890.994823] env[62506]: result = function(*args, **kwargs) [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1890.994823] env[62506]: return func(*args, **kwargs) [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1890.994823] env[62506]: raise e [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1890.994823] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1890.994823] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1890.994823] env[62506]: with excutils.save_and_reraise_exception(): [ 1890.994823] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1890.994823] env[62506]: self.force_reraise() [ 1890.994823] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1890.994823] env[62506]: raise self.value [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1890.994823] env[62506]: updated_port = self._update_port( [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1890.994823] env[62506]: _ensure_no_port_binding_failure(port) [ 1890.994823] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1890.994823] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1890.995530] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1890.995530] env[62506]: Removing descriptor: 15 [ 1890.995530] env[62506]: ERROR nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Traceback (most recent call last): [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] yield resources [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self.driver.spawn(context, instance, image_meta, [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1890.995530] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] vm_ref = self.build_virtual_machine(instance, [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] vif_infos = vmwarevif.get_vif_info(self._session, [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] for vif in network_info: [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return self._sync_wrapper(fn, *args, **kwargs) [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self.wait() [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self[:] = self._gt.wait() [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return self._exit_event.wait() [ 1890.995815] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] result = hub.switch() [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return self.greenlet.switch() [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] result = function(*args, **kwargs) [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return func(*args, **kwargs) [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] raise e [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] nwinfo = self.network_api.allocate_for_instance( [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1890.996166] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] created_port_ids = self._update_ports_for_instance( [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] with excutils.save_and_reraise_exception(): [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self.force_reraise() [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] raise self.value [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] updated_port = self._update_port( [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] _ensure_no_port_binding_failure(port) [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1890.996470] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] raise exception.PortBindingFailed(port_id=port['id']) [ 1890.996742] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1890.996742] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] [ 1890.996742] env[62506]: INFO nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Terminating instance [ 1891.091448] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1891.178770] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1891.274296] env[62506]: DEBUG nova.network.neutron [-] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1891.500301] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquiring lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1891.500494] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquired lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1891.500673] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1891.681653] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Releasing lock "refresh_cache-88322331-7cb9-46a9-8e8c-82f0198c90d8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1891.681933] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1891.682116] env[62506]: DEBUG nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1891.682291] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1891.701215] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1891.777141] env[62506]: INFO nova.compute.manager [-] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Took 1.03 seconds to deallocate network for instance. [ 1891.779277] env[62506]: DEBUG nova.compute.claims [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1891.779462] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1891.889947] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0a31d6c-16d9-4f22-bdd6-5eeb9e8c4f9f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1891.899205] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e59de012-611a-4ab4-b92f-ae8e24a76359 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1891.929822] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59eda3aa-6ebb-4e43-aa8b-706cef7cfd28 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1891.937291] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff13fa0c-9561-4f62-8272-42d25d98b62a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1891.950989] env[62506]: DEBUG nova.compute.provider_tree [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1892.018464] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1892.113864] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1892.203867] env[62506]: DEBUG nova.network.neutron [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1892.348030] env[62506]: DEBUG nova.compute.manager [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Received event network-changed-1359683b-7547-41e9-8cc3-54926ba26963 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1892.348261] env[62506]: DEBUG nova.compute.manager [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Refreshing instance network info cache due to event network-changed-1359683b-7547-41e9-8cc3-54926ba26963. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1892.348457] env[62506]: DEBUG oslo_concurrency.lockutils [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] Acquiring lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1892.454823] env[62506]: DEBUG nova.scheduler.client.report [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1892.615995] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Releasing lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1892.616447] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1892.616647] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1892.616955] env[62506]: DEBUG oslo_concurrency.lockutils [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] Acquired lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1892.617164] env[62506]: DEBUG nova.network.neutron [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Refreshing network info cache for port 1359683b-7547-41e9-8cc3-54926ba26963 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1892.618529] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3d984720-77b0-47d1-8815-df4716d55f98 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1892.627936] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07bc8c13-d4a3-47e0-989b-3b08102a8ea6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1892.649622] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e8433dc4-7c83-4288-acf4-5f5a2863592c could not be found. [ 1892.649824] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1892.650013] env[62506]: INFO nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1892.650302] env[62506]: DEBUG oslo.service.loopingcall [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1892.650530] env[62506]: DEBUG nova.compute.manager [-] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1892.650623] env[62506]: DEBUG nova.network.neutron [-] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1892.666552] env[62506]: DEBUG nova.network.neutron [-] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1892.706811] env[62506]: INFO nova.compute.manager [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] [instance: 88322331-7cb9-46a9-8e8c-82f0198c90d8] Took 1.02 seconds to deallocate network for instance. [ 1892.960487] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.401s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1892.962571] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1892.965148] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 17.304s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1892.965373] env[62506]: DEBUG nova.objects.instance [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] [instance: af042cab-9132-4425-b017-133ec85afa0c] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62506) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 1893.140791] env[62506]: DEBUG nova.network.neutron [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1893.169790] env[62506]: DEBUG nova.network.neutron [-] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1893.240868] env[62506]: DEBUG nova.network.neutron [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1893.470097] env[62506]: DEBUG nova.compute.utils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1893.474429] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1893.474429] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1893.535487] env[62506]: DEBUG nova.policy [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '72828dd561a14a02b21bafd869fcde7f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5bd2b1f6716e473dad9651fd0f68fed8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1893.673214] env[62506]: INFO nova.compute.manager [-] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Took 1.02 seconds to deallocate network for instance. [ 1893.675426] env[62506]: DEBUG nova.compute.claims [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1893.675608] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1893.738907] env[62506]: INFO nova.scheduler.client.report [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Deleted allocations for instance 88322331-7cb9-46a9-8e8c-82f0198c90d8 [ 1893.746518] env[62506]: DEBUG oslo_concurrency.lockutils [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] Releasing lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1893.746760] env[62506]: DEBUG nova.compute.manager [req-07d76c5b-9858-4c8f-8a92-2ab392f0e041 req-9dce318a-eec5-441a-9fae-7370ada88a2f service nova] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Received event network-vif-deleted-1359683b-7547-41e9-8cc3-54926ba26963 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1893.976369] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1893.979557] env[62506]: DEBUG oslo_concurrency.lockutils [None req-77e01986-f143-42c8-98c8-793d667b7b71 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.014s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1893.979863] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.448s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1893.983195] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Successfully created port: dadd5cc2-506e-4d20-80b2-d9eed81b658d {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1894.248869] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e3cf566b-c63b-4ee0-9e44-0f4f630a8bee tempest-DeleteServersAdminTestJSON-16728935 tempest-DeleteServersAdminTestJSON-16728935-project-member] Lock "88322331-7cb9-46a9-8e8c-82f0198c90d8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 82.137s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1894.753289] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1894.884972] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52399e88-033f-49bd-a985-b04a10c24d56 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1894.892610] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa7ac1ec-3f80-47c2-b16c-fcec79c5ed1d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1894.924524] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80676031-4d27-42df-8f3f-2c5a2c6a452d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1894.933394] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b992f550-dd82-4d22-ab0a-78fdc9caf4d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1894.948969] env[62506]: DEBUG nova.compute.provider_tree [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1894.952468] env[62506]: DEBUG nova.compute.manager [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Received event network-changed-dadd5cc2-506e-4d20-80b2-d9eed81b658d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1894.952632] env[62506]: DEBUG nova.compute.manager [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Refreshing instance network info cache due to event network-changed-dadd5cc2-506e-4d20-80b2-d9eed81b658d. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1894.952852] env[62506]: DEBUG oslo_concurrency.lockutils [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] Acquiring lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1894.952994] env[62506]: DEBUG oslo_concurrency.lockutils [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] Acquired lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1894.953166] env[62506]: DEBUG nova.network.neutron [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Refreshing network info cache for port dadd5cc2-506e-4d20-80b2-d9eed81b658d {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1894.992186] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1895.020818] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1895.021147] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1895.021338] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1895.021525] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1895.021670] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1895.021818] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1895.022075] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1895.022264] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1895.022443] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1895.022603] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1895.022776] env[62506]: DEBUG nova.virt.hardware [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1895.024068] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-160366c4-e963-452e-894c-dd669c619a8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1895.032639] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dfd720a-2481-4e28-8147-248039af2fdd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1895.179043] env[62506]: ERROR nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1895.179043] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1895.179043] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1895.179043] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1895.179043] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1895.179043] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1895.179043] env[62506]: ERROR nova.compute.manager raise self.value [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1895.179043] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1895.179043] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1895.179043] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1895.179590] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1895.179590] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1895.179590] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1895.179590] env[62506]: ERROR nova.compute.manager [ 1895.179590] env[62506]: Traceback (most recent call last): [ 1895.179590] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1895.179590] env[62506]: listener.cb(fileno) [ 1895.179590] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1895.179590] env[62506]: result = function(*args, **kwargs) [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1895.179590] env[62506]: return func(*args, **kwargs) [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1895.179590] env[62506]: raise e [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1895.179590] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1895.179590] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1895.179590] env[62506]: with excutils.save_and_reraise_exception(): [ 1895.179590] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1895.179590] env[62506]: self.force_reraise() [ 1895.179590] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1895.179590] env[62506]: raise self.value [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1895.179590] env[62506]: updated_port = self._update_port( [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1895.179590] env[62506]: _ensure_no_port_binding_failure(port) [ 1895.179590] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1895.179590] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1895.180413] env[62506]: nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1895.180413] env[62506]: Removing descriptor: 15 [ 1895.180413] env[62506]: ERROR nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Traceback (most recent call last): [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] yield resources [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self.driver.spawn(context, instance, image_meta, [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1895.180413] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] vm_ref = self.build_virtual_machine(instance, [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] vif_infos = vmwarevif.get_vif_info(self._session, [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] for vif in network_info: [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return self._sync_wrapper(fn, *args, **kwargs) [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self.wait() [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self[:] = self._gt.wait() [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return self._exit_event.wait() [ 1895.180712] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] result = hub.switch() [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return self.greenlet.switch() [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] result = function(*args, **kwargs) [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return func(*args, **kwargs) [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] raise e [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] nwinfo = self.network_api.allocate_for_instance( [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1895.181141] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] created_port_ids = self._update_ports_for_instance( [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] with excutils.save_and_reraise_exception(): [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self.force_reraise() [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] raise self.value [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] updated_port = self._update_port( [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] _ensure_no_port_binding_failure(port) [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1895.181596] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] raise exception.PortBindingFailed(port_id=port['id']) [ 1895.181953] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1895.181953] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] [ 1895.181953] env[62506]: INFO nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Terminating instance [ 1895.275886] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1895.455312] env[62506]: DEBUG nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1895.552765] env[62506]: DEBUG nova.network.neutron [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1895.652480] env[62506]: DEBUG nova.network.neutron [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1895.683345] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquiring lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1895.965023] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.983s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1895.965023] env[62506]: ERROR nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Traceback (most recent call last): [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self.driver.spawn(context, instance, image_meta, [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1895.965023] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] vm_ref = self.build_virtual_machine(instance, [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] vif_infos = vmwarevif.get_vif_info(self._session, [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] for vif in network_info: [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return self._sync_wrapper(fn, *args, **kwargs) [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self.wait() [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self[:] = self._gt.wait() [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return self._exit_event.wait() [ 1895.965589] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] result = hub.switch() [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return self.greenlet.switch() [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] result = function(*args, **kwargs) [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] return func(*args, **kwargs) [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] raise e [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] nwinfo = self.network_api.allocate_for_instance( [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1895.965919] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] created_port_ids = self._update_ports_for_instance( [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] with excutils.save_and_reraise_exception(): [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] self.force_reraise() [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] raise self.value [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] updated_port = self._update_port( [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] _ensure_no_port_binding_failure(port) [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1895.966232] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] raise exception.PortBindingFailed(port_id=port['id']) [ 1895.966576] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] nova.exception.PortBindingFailed: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. [ 1895.966576] env[62506]: ERROR nova.compute.manager [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] [ 1895.966576] env[62506]: DEBUG nova.compute.utils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1895.969585] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.254s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1895.969585] env[62506]: INFO nova.compute.claims [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1895.972864] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Build of instance 6cdfeb65-c53f-4ba6-873c-88e593909878 was re-scheduled: Binding failed for port db27c24c-5cdb-4c11-b330-073ad4c46df0, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1895.973458] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1895.973777] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquiring lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1895.974174] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Acquired lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1895.974435] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1896.158977] env[62506]: DEBUG oslo_concurrency.lockutils [req-464c1a19-41be-444d-a46d-1cb76ddaf241 req-63ead071-20d8-4fcb-b681-711d71ff9bfc service nova] Releasing lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1896.159875] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquired lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1896.159875] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1896.494664] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1896.635221] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1896.677444] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1896.738210] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1896.799085] env[62506]: DEBUG nova.compute.manager [None req-f532ce70-0ad2-42a3-a072-60c83a95cbb4 tempest-ServerDiagnosticsV248Test-1295293030 tempest-ServerDiagnosticsV248Test-1295293030-project-admin] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 1896.799746] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f133a727-fb75-4259-9cad-0f403203cbb2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1896.808201] env[62506]: INFO nova.compute.manager [None req-f532ce70-0ad2-42a3-a072-60c83a95cbb4 tempest-ServerDiagnosticsV248Test-1295293030 tempest-ServerDiagnosticsV248Test-1295293030-project-admin] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Retrieving diagnostics [ 1896.808943] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8d9be9c-402d-48f5-b324-b0954960dbaf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.057552] env[62506]: DEBUG nova.compute.manager [req-13daaeb8-2600-4685-bf0a-f995c46d3631 req-fd1e5358-7bc8-4a7f-b83e-a95c3fa85ef6 service nova] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Received event network-vif-deleted-dadd5cc2-506e-4d20-80b2-d9eed81b658d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1897.137820] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Releasing lock "refresh_cache-6cdfeb65-c53f-4ba6-873c-88e593909878" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1897.138143] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1897.138361] env[62506]: DEBUG nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1897.138534] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1897.157389] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1897.242047] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Releasing lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1897.242047] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1897.242047] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1897.242047] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-77c47361-ace3-483b-9bc4-f4a806e78e50 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.254121] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ef6e2c8-57de-4fd8-bf19-fb17667e2c0a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.281505] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265 could not be found. [ 1897.281744] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1897.281929] env[62506]: INFO nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1897.282190] env[62506]: DEBUG oslo.service.loopingcall [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1897.284868] env[62506]: DEBUG nova.compute.manager [-] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1897.284964] env[62506]: DEBUG nova.network.neutron [-] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1897.306741] env[62506]: DEBUG nova.network.neutron [-] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1897.337374] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquiring lock "7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1897.337603] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Lock "7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1897.338402] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b31cd24-49b4-4a48-9bb7-c17d8b8aba30 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.346721] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd65544e-2792-4b85-a4a3-014e3c4162e8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.376910] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80e59e3f-0451-4116-b669-f3ef5eb079d1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.384778] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bc18e54-b9dc-4aad-910e-e12df14b1a04 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1897.398210] env[62506]: DEBUG nova.compute.provider_tree [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1897.660530] env[62506]: DEBUG nova.network.neutron [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1897.812062] env[62506]: DEBUG nova.network.neutron [-] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1897.901557] env[62506]: DEBUG nova.scheduler.client.report [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1897.969805] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "b4bcab92-0007-4952-8d74-c470049edd71" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1897.970212] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "b4bcab92-0007-4952-8d74-c470049edd71" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1897.970696] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "b4bcab92-0007-4952-8d74-c470049edd71-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1897.970696] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "b4bcab92-0007-4952-8d74-c470049edd71-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1897.970832] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "b4bcab92-0007-4952-8d74-c470049edd71-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1897.973207] env[62506]: INFO nova.compute.manager [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Terminating instance [ 1898.164983] env[62506]: INFO nova.compute.manager [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] [instance: 6cdfeb65-c53f-4ba6-873c-88e593909878] Took 1.02 seconds to deallocate network for instance. [ 1898.314700] env[62506]: INFO nova.compute.manager [-] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Took 1.03 seconds to deallocate network for instance. [ 1898.316887] env[62506]: DEBUG nova.compute.claims [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1898.317212] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1898.406554] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.439s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1898.407086] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1898.409712] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.763s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1898.477130] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "refresh_cache-b4bcab92-0007-4952-8d74-c470049edd71" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1898.477413] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquired lock "refresh_cache-b4bcab92-0007-4952-8d74-c470049edd71" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1898.477516] env[62506]: DEBUG nova.network.neutron [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1898.916946] env[62506]: DEBUG nova.compute.utils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1898.919083] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1898.919284] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1899.001513] env[62506]: DEBUG nova.network.neutron [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1899.010106] env[62506]: DEBUG nova.policy [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a6f8cb335bc541cba4506757f7456b3a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '88ff9f6bfeb148138079fd7bc7c6539d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1899.109323] env[62506]: DEBUG nova.network.neutron [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1899.196838] env[62506]: INFO nova.scheduler.client.report [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Deleted allocations for instance 6cdfeb65-c53f-4ba6-873c-88e593909878 [ 1899.297107] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04ee788e-c33d-4ec0-a80c-d06275cb9186 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.306574] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-050ae521-5947-4022-afa9-ddb37a49b18f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.343232] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2396267d-24d0-4097-93a3-213b5f8fe7cf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.351798] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49870a10-b399-4845-81fb-4e1904f9c385 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.371434] env[62506]: DEBUG nova.compute.provider_tree [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1899.422588] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1899.526595] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Successfully created port: 7549bedb-b15e-4c50-b70a-a32f5a98e486 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1899.615022] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Releasing lock "refresh_cache-b4bcab92-0007-4952-8d74-c470049edd71" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1899.615022] env[62506]: DEBUG nova.compute.manager [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1899.615022] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1899.615022] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a66e831d-8659-4fca-a2f3-2355e166d021 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.625568] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 1899.626292] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-dc693ee8-2d9b-428a-97d2-a08353b70740 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1899.634433] env[62506]: DEBUG oslo_vmware.api [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1899.634433] env[62506]: value = "task-2190821" [ 1899.634433] env[62506]: _type = "Task" [ 1899.634433] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1899.645021] env[62506]: DEBUG oslo_vmware.api [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190821, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1899.708997] env[62506]: DEBUG oslo_concurrency.lockutils [None req-df3c0137-a9bc-4356-be68-f25c15dd560e tempest-ServersTestJSON-639162081 tempest-ServersTestJSON-639162081-project-member] Lock "6cdfeb65-c53f-4ba6-873c-88e593909878" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.187s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1899.876131] env[62506]: DEBUG nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1900.146499] env[62506]: DEBUG oslo_vmware.api [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190821, 'name': PowerOffVM_Task, 'duration_secs': 0.199056} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1900.146806] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 1900.146974] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 1900.147267] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-f576c791-8fe5-4da6-9562-8e98e60cabbf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1900.178431] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 1900.178688] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 1900.178908] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Deleting the datastore file [datastore2] b4bcab92-0007-4952-8d74-c470049edd71 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1900.179198] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-49a1efe1-a559-45d0-acae-5e4743e33183 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1900.188940] env[62506]: DEBUG oslo_vmware.api [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for the task: (returnval){ [ 1900.188940] env[62506]: value = "task-2190823" [ 1900.188940] env[62506]: _type = "Task" [ 1900.188940] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1900.207135] env[62506]: DEBUG oslo_vmware.api [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190823, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1900.215606] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1900.385031] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.973s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1900.385031] env[62506]: ERROR nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Traceback (most recent call last): [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self.driver.spawn(context, instance, image_meta, [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1900.385031] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] vm_ref = self.build_virtual_machine(instance, [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] for vif in network_info: [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return self._sync_wrapper(fn, *args, **kwargs) [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self.wait() [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self[:] = self._gt.wait() [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return self._exit_event.wait() [ 1900.385563] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] result = hub.switch() [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return self.greenlet.switch() [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] result = function(*args, **kwargs) [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] return func(*args, **kwargs) [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] raise e [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] nwinfo = self.network_api.allocate_for_instance( [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1900.385866] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] created_port_ids = self._update_ports_for_instance( [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] with excutils.save_and_reraise_exception(): [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] self.force_reraise() [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] raise self.value [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] updated_port = self._update_port( [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] _ensure_no_port_binding_failure(port) [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1900.386193] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] raise exception.PortBindingFailed(port_id=port['id']) [ 1900.386469] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] nova.exception.PortBindingFailed: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. [ 1900.386469] env[62506]: ERROR nova.compute.manager [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] [ 1900.386469] env[62506]: DEBUG nova.compute.utils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1900.390021] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 15.489s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1900.390021] env[62506]: DEBUG nova.objects.instance [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] [instance: af042cab-9132-4425-b017-133ec85afa0c] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62506) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 1900.391894] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Build of instance 249e5265-86b3-4ba8-96b9-d21ea0a5599d was re-scheduled: Binding failed for port ed9e4982-0738-4d5a-bbcb-4ec0b9950baa, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1900.392557] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1900.392915] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquiring lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1900.393224] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Acquired lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1900.393648] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1900.432882] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1900.466919] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1900.466919] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1900.466919] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1900.468329] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1900.468329] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1900.468329] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1900.468329] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1900.468329] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1900.468787] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1900.468787] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1900.468787] env[62506]: DEBUG nova.virt.hardware [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1900.469726] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d4069de-062b-410f-9d88-2cf9c4daeda8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1900.478164] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae6e28f8-8cc6-4261-9cf3-2ca35f7f27bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1900.532172] env[62506]: DEBUG nova.compute.manager [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Received event network-changed-7549bedb-b15e-4c50-b70a-a32f5a98e486 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1900.532172] env[62506]: DEBUG nova.compute.manager [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Refreshing instance network info cache due to event network-changed-7549bedb-b15e-4c50-b70a-a32f5a98e486. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1900.532172] env[62506]: DEBUG oslo_concurrency.lockutils [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] Acquiring lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1900.532172] env[62506]: DEBUG oslo_concurrency.lockutils [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] Acquired lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1900.532172] env[62506]: DEBUG nova.network.neutron [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Refreshing network info cache for port 7549bedb-b15e-4c50-b70a-a32f5a98e486 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1900.698440] env[62506]: DEBUG oslo_vmware.api [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Task: {'id': task-2190823, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.203324} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1900.698790] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1900.699943] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 1900.699943] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1900.699943] env[62506]: INFO nova.compute.manager [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Took 1.09 seconds to destroy the instance on the hypervisor. [ 1900.699943] env[62506]: DEBUG oslo.service.loopingcall [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1900.699943] env[62506]: DEBUG nova.compute.manager [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1900.699943] env[62506]: DEBUG nova.network.neutron [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1900.728171] env[62506]: DEBUG nova.network.neutron [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1900.751928] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1900.785593] env[62506]: ERROR nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1900.785593] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1900.785593] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1900.785593] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1900.785593] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1900.785593] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1900.785593] env[62506]: ERROR nova.compute.manager raise self.value [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1900.785593] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1900.785593] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1900.785593] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1900.787196] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1900.787196] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1900.787196] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1900.787196] env[62506]: ERROR nova.compute.manager [ 1900.787196] env[62506]: Traceback (most recent call last): [ 1900.787196] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1900.787196] env[62506]: listener.cb(fileno) [ 1900.787196] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1900.787196] env[62506]: result = function(*args, **kwargs) [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1900.787196] env[62506]: return func(*args, **kwargs) [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1900.787196] env[62506]: raise e [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1900.787196] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1900.787196] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1900.787196] env[62506]: with excutils.save_and_reraise_exception(): [ 1900.787196] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1900.787196] env[62506]: self.force_reraise() [ 1900.787196] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1900.787196] env[62506]: raise self.value [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1900.787196] env[62506]: updated_port = self._update_port( [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1900.787196] env[62506]: _ensure_no_port_binding_failure(port) [ 1900.787196] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1900.787196] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1900.788336] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1900.788336] env[62506]: Removing descriptor: 15 [ 1900.788336] env[62506]: ERROR nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Traceback (most recent call last): [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] yield resources [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self.driver.spawn(context, instance, image_meta, [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1900.788336] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] vm_ref = self.build_virtual_machine(instance, [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] for vif in network_info: [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return self._sync_wrapper(fn, *args, **kwargs) [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self.wait() [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self[:] = self._gt.wait() [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return self._exit_event.wait() [ 1900.788853] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] result = hub.switch() [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return self.greenlet.switch() [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] result = function(*args, **kwargs) [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return func(*args, **kwargs) [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] raise e [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] nwinfo = self.network_api.allocate_for_instance( [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1900.789405] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] created_port_ids = self._update_ports_for_instance( [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] with excutils.save_and_reraise_exception(): [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self.force_reraise() [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] raise self.value [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] updated_port = self._update_port( [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] _ensure_no_port_binding_failure(port) [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1900.789862] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] raise exception.PortBindingFailed(port_id=port['id']) [ 1900.791421] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1900.791421] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] [ 1900.791421] env[62506]: INFO nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Terminating instance [ 1900.913096] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1901.002034] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1901.048929] env[62506]: DEBUG nova.network.neutron [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1901.151564] env[62506]: DEBUG nova.network.neutron [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1901.233359] env[62506]: DEBUG nova.network.neutron [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1901.292041] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1901.400020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6c402b20-eacf-4ac3-b082-3377ea8ae8f7 tempest-ServersAdmin275Test-1453848816 tempest-ServersAdmin275Test-1453848816-project-admin] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.010s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1901.400020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.906s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1901.504696] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Releasing lock "refresh_cache-249e5265-86b3-4ba8-96b9-d21ea0a5599d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1901.507026] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1901.507026] env[62506]: DEBUG nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1901.507026] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1901.522620] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1901.654626] env[62506]: DEBUG oslo_concurrency.lockutils [req-70c42803-ae2e-4e1b-8d75-be6f49692c18 req-c7270b62-5b4e-4f7a-b438-6319cc77bc24 service nova] Releasing lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1901.655077] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquired lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1901.655276] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1901.736979] env[62506]: INFO nova.compute.manager [-] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Took 1.04 seconds to deallocate network for instance. [ 1902.024799] env[62506]: DEBUG nova.network.neutron [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1902.179172] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1902.236957] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37ddb8dc-4d32-4695-865f-619d1f607f27 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1902.246112] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00d6c414-22f5-45dc-8309-34a3a744300e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1902.249560] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1902.281123] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3b65490-1e68-40cb-a141-f5746609a949 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1902.289490] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44856f05-b388-459a-98ad-69746e387aa2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1902.304293] env[62506]: DEBUG nova.compute.provider_tree [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1902.447040] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1902.527068] env[62506]: INFO nova.compute.manager [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] [instance: 249e5265-86b3-4ba8-96b9-d21ea0a5599d] Took 1.02 seconds to deallocate network for instance. [ 1902.565458] env[62506]: DEBUG nova.compute.manager [req-a2892ef5-139b-4e25-a5e8-d71d9864bef0 req-47114543-9910-4ac9-8f21-5d5f81508586 service nova] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Received event network-vif-deleted-7549bedb-b15e-4c50-b70a-a32f5a98e486 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1902.808315] env[62506]: DEBUG nova.scheduler.client.report [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1902.950229] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Releasing lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1902.950753] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1902.950950] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1902.951281] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9b87edf1-a377-414f-a50a-2615af220f5b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1902.960194] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb2926b5-e489-43f9-9e9c-07c5669d1154 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1902.981503] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7e185b7e-8a95-4931-96a8-1c27516f767d could not be found. [ 1902.981725] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1902.981907] env[62506]: INFO nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1902.982159] env[62506]: DEBUG oslo.service.loopingcall [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1902.982377] env[62506]: DEBUG nova.compute.manager [-] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1902.982518] env[62506]: DEBUG nova.network.neutron [-] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1902.998756] env[62506]: DEBUG nova.network.neutron [-] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1903.321294] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.922s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1903.321978] env[62506]: ERROR nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Traceback (most recent call last): [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self.driver.spawn(context, instance, image_meta, [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] vm_ref = self.build_virtual_machine(instance, [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] vif_infos = vmwarevif.get_vif_info(self._session, [ 1903.321978] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] for vif in network_info: [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return self._sync_wrapper(fn, *args, **kwargs) [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self.wait() [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self[:] = self._gt.wait() [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return self._exit_event.wait() [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] result = hub.switch() [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1903.322363] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return self.greenlet.switch() [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] result = function(*args, **kwargs) [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] return func(*args, **kwargs) [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] raise e [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] nwinfo = self.network_api.allocate_for_instance( [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] created_port_ids = self._update_ports_for_instance( [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] with excutils.save_and_reraise_exception(): [ 1903.322795] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] self.force_reraise() [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] raise self.value [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] updated_port = self._update_port( [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] _ensure_no_port_binding_failure(port) [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] raise exception.PortBindingFailed(port_id=port['id']) [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] nova.exception.PortBindingFailed: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. [ 1903.323192] env[62506]: ERROR nova.compute.manager [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] [ 1903.323496] env[62506]: DEBUG nova.compute.utils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1903.324068] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.424s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1903.325720] env[62506]: INFO nova.compute.claims [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1903.329835] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Build of instance a7ed3eba-aced-4ff9-ba98-e29d44ef5c54 was re-scheduled: Binding failed for port e58bb70a-f560-4376-92b6-ae80163c2364, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1903.329835] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1903.329835] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquiring lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1903.329835] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Acquired lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1903.329990] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1903.500956] env[62506]: DEBUG nova.network.neutron [-] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1903.565562] env[62506]: INFO nova.scheduler.client.report [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Deleted allocations for instance 249e5265-86b3-4ba8-96b9-d21ea0a5599d [ 1903.853148] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1903.973686] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1904.003578] env[62506]: INFO nova.compute.manager [-] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Took 1.02 seconds to deallocate network for instance. [ 1904.007876] env[62506]: DEBUG nova.compute.claims [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1904.008075] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1904.079824] env[62506]: DEBUG oslo_concurrency.lockutils [None req-be3eff24-a5ac-4766-ba70-3175b10c4b75 tempest-ServerPasswordTestJSON-2097700010 tempest-ServerPasswordTestJSON-2097700010-project-member] Lock "249e5265-86b3-4ba8-96b9-d21ea0a5599d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 88.313s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1904.476992] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Releasing lock "refresh_cache-a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1904.477280] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1904.477426] env[62506]: DEBUG nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1904.477586] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1904.579806] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1904.585858] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1904.730827] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10425ec0-148a-4f1c-af40-0fc5cafded36 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1904.738367] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c91aa69-3a33-4369-9b80-56e0ba9f28fe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1904.771235] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-746f80a6-a59e-45c8-9f3a-42a5b71b235c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1904.780046] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9ce07cb-caef-4073-95ff-36006dde3b92 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1904.794036] env[62506]: DEBUG nova.compute.provider_tree [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1905.081163] env[62506]: DEBUG nova.network.neutron [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1905.105229] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1905.297166] env[62506]: DEBUG nova.scheduler.client.report [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1905.584644] env[62506]: INFO nova.compute.manager [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] [instance: a7ed3eba-aced-4ff9-ba98-e29d44ef5c54] Took 1.11 seconds to deallocate network for instance. [ 1905.802135] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.478s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1905.802404] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1905.807481] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.401s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1905.811018] env[62506]: INFO nova.compute.claims [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1906.319814] env[62506]: DEBUG nova.compute.utils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1906.321504] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1906.321706] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1906.379427] env[62506]: DEBUG nova.policy [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5df8fc65cc014ce69d7319a75cc2c45e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '49e5b835174f450a84ad9f07457b5638', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1906.626361] env[62506]: INFO nova.scheduler.client.report [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Deleted allocations for instance a7ed3eba-aced-4ff9-ba98-e29d44ef5c54 [ 1906.826786] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1906.858988] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Successfully created port: d015a181-70f8-4b40-bef7-e581cbfb668d {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1906.931674] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquiring lock "c1943459-83bf-4461-80e0-d98da37b5d17" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1906.931897] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Lock "c1943459-83bf-4461-80e0-d98da37b5d17" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1907.137306] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d7955ed5-8064-4371-b4bc-3e7d8aaa7bcc tempest-AttachInterfacesV270Test-1359339946 tempest-AttachInterfacesV270Test-1359339946-project-member] Lock "a7ed3eba-aced-4ff9-ba98-e29d44ef5c54" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.835s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1907.217448] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e6d37b6-766d-4a13-9621-7fa672279c76 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.225907] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a6c3fef-8278-4893-8e65-2528a332e540 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.258457] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbee3e48-da0f-4211-9308-ddc01d3a68f6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.266076] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec28832e-a501-469e-a358-1ae764e880cc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.278910] env[62506]: DEBUG nova.compute.provider_tree [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1907.640043] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1907.785109] env[62506]: DEBUG nova.scheduler.client.report [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1907.836400] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1907.864102] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1907.864489] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1907.864562] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1907.864748] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1907.864896] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1907.865051] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1907.865269] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1907.865422] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1907.865592] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1907.865755] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1907.865929] env[62506]: DEBUG nova.virt.hardware [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1907.866814] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8afb1c43-869f-4d02-a202-89b620a474c2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.874835] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e472802d-4b16-4134-a483-e3000b55cf69 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.882054] env[62506]: DEBUG nova.compute.manager [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Received event network-changed-d015a181-70f8-4b40-bef7-e581cbfb668d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1907.882161] env[62506]: DEBUG nova.compute.manager [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Refreshing instance network info cache due to event network-changed-d015a181-70f8-4b40-bef7-e581cbfb668d. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1907.882349] env[62506]: DEBUG oslo_concurrency.lockutils [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] Acquiring lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1907.883092] env[62506]: DEBUG oslo_concurrency.lockutils [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] Acquired lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1907.883278] env[62506]: DEBUG nova.network.neutron [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Refreshing network info cache for port d015a181-70f8-4b40-bef7-e581cbfb668d {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1907.956600] env[62506]: DEBUG nova.network.neutron [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1908.106627] env[62506]: ERROR nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1908.106627] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1908.106627] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1908.106627] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1908.106627] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1908.106627] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1908.106627] env[62506]: ERROR nova.compute.manager raise self.value [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1908.106627] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1908.106627] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1908.106627] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1908.107104] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1908.107104] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1908.107104] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1908.107104] env[62506]: ERROR nova.compute.manager [ 1908.107104] env[62506]: Traceback (most recent call last): [ 1908.107104] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1908.107104] env[62506]: listener.cb(fileno) [ 1908.107104] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1908.107104] env[62506]: result = function(*args, **kwargs) [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1908.107104] env[62506]: return func(*args, **kwargs) [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1908.107104] env[62506]: raise e [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1908.107104] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1908.107104] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1908.107104] env[62506]: with excutils.save_and_reraise_exception(): [ 1908.107104] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1908.107104] env[62506]: self.force_reraise() [ 1908.107104] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1908.107104] env[62506]: raise self.value [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1908.107104] env[62506]: updated_port = self._update_port( [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1908.107104] env[62506]: _ensure_no_port_binding_failure(port) [ 1908.107104] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1908.107104] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1908.107792] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1908.107792] env[62506]: Removing descriptor: 15 [ 1908.107792] env[62506]: ERROR nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Traceback (most recent call last): [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] yield resources [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self.driver.spawn(context, instance, image_meta, [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1908.107792] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] vm_ref = self.build_virtual_machine(instance, [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] vif_infos = vmwarevif.get_vif_info(self._session, [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] for vif in network_info: [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return self._sync_wrapper(fn, *args, **kwargs) [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self.wait() [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self[:] = self._gt.wait() [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return self._exit_event.wait() [ 1908.108100] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] result = hub.switch() [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return self.greenlet.switch() [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] result = function(*args, **kwargs) [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return func(*args, **kwargs) [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] raise e [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] nwinfo = self.network_api.allocate_for_instance( [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1908.108419] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] created_port_ids = self._update_ports_for_instance( [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] with excutils.save_and_reraise_exception(): [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self.force_reraise() [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] raise self.value [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] updated_port = self._update_port( [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] _ensure_no_port_binding_failure(port) [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1908.108730] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] raise exception.PortBindingFailed(port_id=port['id']) [ 1908.109025] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1908.109025] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] [ 1908.109025] env[62506]: INFO nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Terminating instance [ 1908.117368] env[62506]: DEBUG nova.network.neutron [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1908.161962] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1908.291847] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.485s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1908.292395] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1908.295433] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 19.604s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1908.611738] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquiring lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1908.620844] env[62506]: DEBUG oslo_concurrency.lockutils [req-818d01ac-8d3e-4c16-b528-2120eb5b5f69 req-6b815199-cea0-4a79-a6e0-ab17dafb557b service nova] Releasing lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1908.620844] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquired lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1908.620844] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1908.798365] env[62506]: DEBUG nova.compute.utils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1908.799980] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1908.800190] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1908.848913] env[62506]: DEBUG nova.policy [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a6f8cb335bc541cba4506757f7456b3a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '88ff9f6bfeb148138079fd7bc7c6539d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1909.160129] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1909.312332] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1909.318585] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1909.332768] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Successfully created port: 121430c8-695c-4a4f-bbeb-62fec4457c07 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1909.347206] env[62506]: WARNING nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance af042cab-9132-4425-b017-133ec85afa0c is not being actively managed by this compute host but has allocations referencing this compute host: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. Skipping heal of allocation because we do not know what to do. [ 1909.347314] env[62506]: WARNING nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b4bcab92-0007-4952-8d74-c470049edd71 is not being actively managed by this compute host but has allocations referencing this compute host: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. Skipping heal of allocation because we do not know what to do. [ 1909.347356] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8fd213dd-d597-48c0-9387-19b4f5dd72ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1909.348242] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e8433dc4-7c83-4288-acf4-5f5a2863592c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1909.348242] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1909.348242] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7e185b7e-8a95-4931-96a8-1c27516f767d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1909.348242] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e0de75e0-ca8e-4525-ae7f-ed13fcd07338 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1909.348433] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 03e1ebe5-7a02-430e-b38c-d215e85d59ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1909.826216] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Releasing lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1909.826557] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1909.826752] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1909.827263] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-82dc2d0d-e0e0-43c7-8c85-4738a867fa5b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1909.836713] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d00ad57-1c45-4f96-b89c-7f5cab7bd960 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1909.850985] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 43d09ce7-7946-43d1-8729-d7068c1cdec4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1909.864357] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e0de75e0-ca8e-4525-ae7f-ed13fcd07338 could not be found. [ 1909.864574] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1909.864759] env[62506]: INFO nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1909.865013] env[62506]: DEBUG oslo.service.loopingcall [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1909.866337] env[62506]: DEBUG nova.compute.manager [-] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1909.866471] env[62506]: DEBUG nova.network.neutron [-] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1909.889138] env[62506]: DEBUG nova.network.neutron [-] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1909.907236] env[62506]: DEBUG nova.compute.manager [req-cb0d4855-f29f-41a9-814d-1dea2ec97f0e req-e7b27680-5fd5-47cd-b4b8-37f24f5f8a38 service nova] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Received event network-vif-deleted-d015a181-70f8-4b40-bef7-e581cbfb668d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1910.329900] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1910.356195] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1910.364543] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1910.365125] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1910.368243] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1910.368471] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1910.368656] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1910.368884] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1910.369154] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1910.369360] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1910.369575] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1910.369786] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1910.370236] env[62506]: DEBUG nova.virt.hardware [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1910.371144] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b26745dd-8fdc-418b-8d6d-f4e5951273c7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1910.382373] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5456c9b4-0e57-4eb8-a671-1967662b2aa7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1910.395114] env[62506]: DEBUG nova.network.neutron [-] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1910.674360] env[62506]: ERROR nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1910.674360] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1910.674360] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1910.674360] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1910.674360] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1910.674360] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1910.674360] env[62506]: ERROR nova.compute.manager raise self.value [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1910.674360] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1910.674360] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1910.674360] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1910.674857] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1910.674857] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1910.674857] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1910.674857] env[62506]: ERROR nova.compute.manager [ 1910.674857] env[62506]: Traceback (most recent call last): [ 1910.674857] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1910.674857] env[62506]: listener.cb(fileno) [ 1910.674857] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1910.674857] env[62506]: result = function(*args, **kwargs) [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1910.674857] env[62506]: return func(*args, **kwargs) [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1910.674857] env[62506]: raise e [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1910.674857] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1910.674857] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1910.674857] env[62506]: with excutils.save_and_reraise_exception(): [ 1910.674857] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1910.674857] env[62506]: self.force_reraise() [ 1910.674857] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1910.674857] env[62506]: raise self.value [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1910.674857] env[62506]: updated_port = self._update_port( [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1910.674857] env[62506]: _ensure_no_port_binding_failure(port) [ 1910.674857] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1910.674857] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1910.675771] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1910.675771] env[62506]: Removing descriptor: 19 [ 1910.675771] env[62506]: ERROR nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Traceback (most recent call last): [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] yield resources [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self.driver.spawn(context, instance, image_meta, [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1910.675771] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] vm_ref = self.build_virtual_machine(instance, [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] vif_infos = vmwarevif.get_vif_info(self._session, [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] for vif in network_info: [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return self._sync_wrapper(fn, *args, **kwargs) [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self.wait() [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self[:] = self._gt.wait() [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return self._exit_event.wait() [ 1910.676085] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] result = hub.switch() [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return self.greenlet.switch() [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] result = function(*args, **kwargs) [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return func(*args, **kwargs) [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] raise e [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] nwinfo = self.network_api.allocate_for_instance( [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1910.676446] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] created_port_ids = self._update_ports_for_instance( [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] with excutils.save_and_reraise_exception(): [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self.force_reraise() [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] raise self.value [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] updated_port = self._update_port( [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] _ensure_no_port_binding_failure(port) [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1910.676884] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] raise exception.PortBindingFailed(port_id=port['id']) [ 1910.677247] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1910.677247] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] [ 1910.677247] env[62506]: INFO nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Terminating instance [ 1910.860139] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 01ad7fbe-7ea0-425b-ba63-28537cb158b1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1910.898273] env[62506]: INFO nova.compute.manager [-] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Took 1.03 seconds to deallocate network for instance. [ 1910.900724] env[62506]: DEBUG nova.compute.claims [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1910.900936] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1911.180516] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1911.180724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquired lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1911.180910] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1911.362386] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 854c955f-b327-4c25-87bd-3ad68dbff024 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1911.708216] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1911.832016] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1911.866957] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 6cdba5b2-af96-4766-9a97-3a758983c49b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1911.953878] env[62506]: DEBUG nova.compute.manager [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Received event network-changed-121430c8-695c-4a4f-bbeb-62fec4457c07 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1911.954103] env[62506]: DEBUG nova.compute.manager [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Refreshing instance network info cache due to event network-changed-121430c8-695c-4a4f-bbeb-62fec4457c07. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1911.954306] env[62506]: DEBUG oslo_concurrency.lockutils [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] Acquiring lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1912.334577] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Releasing lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1912.335053] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1912.335269] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1912.335617] env[62506]: DEBUG oslo_concurrency.lockutils [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] Acquired lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1912.335769] env[62506]: DEBUG nova.network.neutron [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Refreshing network info cache for port 121430c8-695c-4a4f-bbeb-62fec4457c07 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1912.336854] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4768e392-7257-432b-9c3b-3b4baab578f5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1912.346821] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b7f1f83-1ded-4da8-bc77-310d8e221f2a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1912.368086] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 03e1ebe5-7a02-430e-b38c-d215e85d59ae could not be found. [ 1912.368293] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1912.368490] env[62506]: INFO nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1912.368760] env[62506]: DEBUG oslo.service.loopingcall [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1912.368933] env[62506]: DEBUG nova.compute.manager [-] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1912.369040] env[62506]: DEBUG nova.network.neutron [-] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1912.370934] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 08cc6c54-48f2-4064-93a4-4debb98ab606 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1912.386930] env[62506]: DEBUG nova.network.neutron [-] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1912.859943] env[62506]: DEBUG nova.network.neutron [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1912.874707] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 3d034f40-5896-49bc-bed1-9062e1fde1e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1912.889626] env[62506]: DEBUG nova.network.neutron [-] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1912.960715] env[62506]: DEBUG nova.network.neutron [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1913.377901] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b5d6b647-e24f-4e53-b448-0614334fd760 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1913.394538] env[62506]: INFO nova.compute.manager [-] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Took 1.03 seconds to deallocate network for instance. [ 1913.396516] env[62506]: DEBUG nova.compute.claims [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1913.396694] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1913.463977] env[62506]: DEBUG oslo_concurrency.lockutils [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] Releasing lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1913.464266] env[62506]: DEBUG nova.compute.manager [req-833237ea-d230-4b30-bd97-580cc34278f4 req-dc3fc32d-bdfa-4994-9ebd-e723a995f16e service nova] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Received event network-vif-deleted-121430c8-695c-4a4f-bbeb-62fec4457c07 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1913.880889] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dcd5a800-1a28-45b3-8eff-a2e8cb871491 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1914.383723] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 04218b42-91dd-4045-8a40-f8829910e064 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1914.886358] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance ecc8aab4-a172-4218-8b09-ccfde0d03903 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1915.389508] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 19afb920-58b0-4ad9-be99-d0a8e930c680 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1915.894032] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dc1c594f-7625-495a-8948-cee046bf204a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1916.396104] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e3c3759c-81e6-415d-a5a5-b17cc16bdc2f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1916.898862] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8f808943-6107-49d9-966f-c4e6ae0cdac4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1917.402923] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1917.906428] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance c1943459-83bf-4461-80e0-d98da37b5d17 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1917.906726] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1917.906822] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1918.189889] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b48b2109-a7f8-4660-adbe-b1f4b3a07312 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1918.197391] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3609729-d6aa-4bb9-8d2e-9cc442ef056e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1918.226520] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c25ca933-1ecf-4a38-93c4-3f031ddfea68 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1918.233533] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f7000dd-b0b7-4fe7-99f9-f234287ad427 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1918.246827] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1918.750430] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1919.256277] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1919.256673] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 10.962s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1919.257107] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 29.233s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1919.257415] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1919.260469] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 27.481s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1919.282869] env[62506]: INFO nova.scheduler.client.report [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Deleted allocations for instance af042cab-9132-4425-b017-133ec85afa0c [ 1919.792637] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fe5d22a4-d07e-40f7-a7c3-9db710edc995 tempest-ServersAdmin275Test-991344367 tempest-ServersAdmin275Test-991344367-project-member] Lock "af042cab-9132-4425-b017-133ec85afa0c" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 34.090s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1920.057586] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-631d6325-6aac-4f08-bd79-01efd2a040df {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1920.066068] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1f44207-f2df-4af0-a1df-d0fe1b6655ae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1920.098481] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-844af74c-cab3-4f27-b515-238d802779a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1920.106168] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-439503eb-3215-423c-9968-c4b4e49ffe3d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1920.119625] env[62506]: DEBUG nova.compute.provider_tree [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1920.261055] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1920.261055] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1920.261055] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1920.261550] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1920.622797] env[62506]: DEBUG nova.scheduler.client.report [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1920.766055] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1920.766055] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1920.766224] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1920.766224] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1920.766332] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1920.780351] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "refresh_cache-b4bcab92-0007-4952-8d74-c470049edd71" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1920.780570] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquired lock "refresh_cache-b4bcab92-0007-4952-8d74-c470049edd71" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1920.780719] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Forcefully refreshing network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2061}} [ 1920.780874] env[62506]: DEBUG nova.objects.instance [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lazy-loading 'info_cache' on Instance uuid b4bcab92-0007-4952-8d74-c470049edd71 {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 1921.127407] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.867s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1921.128285] env[62506]: ERROR nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Traceback (most recent call last): [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self.driver.spawn(context, instance, image_meta, [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] vm_ref = self.build_virtual_machine(instance, [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] vif_infos = vmwarevif.get_vif_info(self._session, [ 1921.128285] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] for vif in network_info: [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return self._sync_wrapper(fn, *args, **kwargs) [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self.wait() [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self[:] = self._gt.wait() [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return self._exit_event.wait() [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] result = hub.switch() [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1921.128723] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return self.greenlet.switch() [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] result = function(*args, **kwargs) [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] return func(*args, **kwargs) [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] raise e [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] nwinfo = self.network_api.allocate_for_instance( [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] created_port_ids = self._update_ports_for_instance( [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] with excutils.save_and_reraise_exception(): [ 1921.129105] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] self.force_reraise() [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] raise self.value [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] updated_port = self._update_port( [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] _ensure_no_port_binding_failure(port) [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] raise exception.PortBindingFailed(port_id=port['id']) [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] nova.exception.PortBindingFailed: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. [ 1921.129424] env[62506]: ERROR nova.compute.manager [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] [ 1921.129690] env[62506]: DEBUG nova.compute.utils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1921.130234] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 27.455s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1921.133394] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Build of instance 8fd213dd-d597-48c0-9387-19b4f5dd72ff was re-scheduled: Binding failed for port cbff7b5a-bd00-4b02-b8f4-3cf66d1ce9dd, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1921.133857] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1921.134095] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1921.134245] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1921.134403] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1921.663902] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1921.986514] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1922.021640] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1922.031807] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a3dc079-b72e-4882-871f-3827f1023112 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1922.040482] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb5bd66a-1dc5-43f4-929f-b96952374704 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1922.074097] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-655ee8c0-ec20-4621-8bf9-d41091a42056 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1922.081387] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01d4e5b2-41ff-4f30-aaf4-b8d68f5bca76 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1922.094730] env[62506]: DEBUG nova.compute.provider_tree [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1922.525924] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-8fd213dd-d597-48c0-9387-19b4f5dd72ff" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1922.526186] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1922.526371] env[62506]: DEBUG nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1922.526536] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1922.553432] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1922.601542] env[62506]: DEBUG nova.scheduler.client.report [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1922.614985] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1923.055596] env[62506]: DEBUG nova.network.neutron [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1923.107246] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.977s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1923.107902] env[62506]: ERROR nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Traceback (most recent call last): [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self.driver.spawn(context, instance, image_meta, [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] vm_ref = self.build_virtual_machine(instance, [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] vif_infos = vmwarevif.get_vif_info(self._session, [ 1923.107902] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] for vif in network_info: [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return self._sync_wrapper(fn, *args, **kwargs) [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self.wait() [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self[:] = self._gt.wait() [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return self._exit_event.wait() [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] result = hub.switch() [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1923.108498] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return self.greenlet.switch() [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] result = function(*args, **kwargs) [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] return func(*args, **kwargs) [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] raise e [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] nwinfo = self.network_api.allocate_for_instance( [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] created_port_ids = self._update_ports_for_instance( [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] with excutils.save_and_reraise_exception(): [ 1923.109045] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] self.force_reraise() [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] raise self.value [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] updated_port = self._update_port( [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] _ensure_no_port_binding_failure(port) [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] raise exception.PortBindingFailed(port_id=port['id']) [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] nova.exception.PortBindingFailed: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. [ 1923.109577] env[62506]: ERROR nova.compute.manager [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] [ 1923.110043] env[62506]: DEBUG nova.compute.utils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1923.110610] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Build of instance e8433dc4-7c83-4288-acf4-5f5a2863592c was re-scheduled: Binding failed for port 1359683b-7547-41e9-8cc3-54926ba26963, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1923.111041] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1923.111274] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquiring lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1923.111448] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Acquired lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1923.111623] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1923.113136] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.837s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1923.114586] env[62506]: INFO nova.compute.claims [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1923.118372] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Releasing lock "refresh_cache-b4bcab92-0007-4952-8d74-c470049edd71" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1923.118544] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Updated the network info_cache for instance {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10516}} [ 1923.118728] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1923.118893] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1923.119096] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1923.560688] env[62506]: INFO nova.compute.manager [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 8fd213dd-d597-48c0-9387-19b4f5dd72ff] Took 1.03 seconds to deallocate network for instance. [ 1923.632177] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1923.802540] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1924.305466] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Releasing lock "refresh_cache-e8433dc4-7c83-4288-acf4-5f5a2863592c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1924.305724] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1924.305724] env[62506]: DEBUG nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1924.305873] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1924.331242] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1924.472104] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad63db81-d015-4979-813e-c533b6388cd6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1924.481523] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bac94640-156f-4be3-a204-2672f727fef9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1924.523457] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12461abb-5d25-40b3-bc76-864e2c7e1367 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1924.533416] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f051181-9714-48c8-b0ad-731f4b544e00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1924.549789] env[62506]: DEBUG nova.compute.provider_tree [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1924.593808] env[62506]: INFO nova.scheduler.client.report [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Deleted allocations for instance 8fd213dd-d597-48c0-9387-19b4f5dd72ff [ 1924.834768] env[62506]: DEBUG nova.network.neutron [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1925.051169] env[62506]: DEBUG nova.scheduler.client.report [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1925.106654] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5fa4c11a-38f9-46c8-ac56-9c09b185d2f5 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "8fd213dd-d597-48c0-9387-19b4f5dd72ff" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 107.463s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1925.338765] env[62506]: INFO nova.compute.manager [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] [instance: e8433dc4-7c83-4288-acf4-5f5a2863592c] Took 1.03 seconds to deallocate network for instance. [ 1925.556015] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.443s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1925.558177] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1925.563164] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 27.242s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1925.608833] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1925.613383] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1926.058553] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquiring lock "0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1926.058822] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Lock "0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1926.067801] env[62506]: DEBUG nova.compute.utils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1926.072526] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1926.072900] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1926.117113] env[62506]: DEBUG nova.policy [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e75dcde364f4499b914f7b714e93bf64', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '232cfc6eccef4af5a84e30a34403b5b8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1926.138124] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1926.367608] env[62506]: INFO nova.scheduler.client.report [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Deleted allocations for instance e8433dc4-7c83-4288-acf4-5f5a2863592c [ 1926.390553] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45789585-9b66-4cf4-a63c-9f7df6666403 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.399506] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50651a35-ed8c-49a9-96fd-f16e464177ac {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.436069] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aaf3b709-cf4b-455d-aa52-bc8f217312f5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.443820] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b0f8fca-7974-42fc-a705-6d68cafefe00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.459197] env[62506]: DEBUG nova.compute.provider_tree [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1926.570413] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1926.693921] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Successfully created port: b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1926.724089] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "85f24cb7-a0c3-4598-9992-2726320ca76d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1926.724344] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "85f24cb7-a0c3-4598-9992-2726320ca76d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1926.878170] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f41d0018-918d-4f73-8cee-d1867c111225 tempest-ServerAddressesNegativeTestJSON-521960601 tempest-ServerAddressesNegativeTestJSON-521960601-project-member] Lock "e8433dc4-7c83-4288-acf4-5f5a2863592c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 100.992s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1926.964010] env[62506]: DEBUG nova.scheduler.client.report [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1927.382495] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1927.467342] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.908s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1927.468033] env[62506]: ERROR nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Traceback (most recent call last): [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self.driver.spawn(context, instance, image_meta, [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] vm_ref = self.build_virtual_machine(instance, [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] vif_infos = vmwarevif.get_vif_info(self._session, [ 1927.468033] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] for vif in network_info: [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return self._sync_wrapper(fn, *args, **kwargs) [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self.wait() [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self[:] = self._gt.wait() [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return self._exit_event.wait() [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] result = hub.switch() [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1927.468403] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return self.greenlet.switch() [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] result = function(*args, **kwargs) [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] return func(*args, **kwargs) [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] raise e [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] nwinfo = self.network_api.allocate_for_instance( [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] created_port_ids = self._update_ports_for_instance( [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] with excutils.save_and_reraise_exception(): [ 1927.468774] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] self.force_reraise() [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] raise self.value [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] updated_port = self._update_port( [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] _ensure_no_port_binding_failure(port) [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] raise exception.PortBindingFailed(port_id=port['id']) [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] nova.exception.PortBindingFailed: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. [ 1927.469181] env[62506]: ERROR nova.compute.manager [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] [ 1927.469511] env[62506]: DEBUG nova.compute.utils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1927.469867] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.718s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1927.471407] env[62506]: INFO nova.compute.claims [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1927.473861] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Build of instance 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265 was re-scheduled: Binding failed for port dadd5cc2-506e-4d20-80b2-d9eed81b658d, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1927.474365] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1927.474584] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquiring lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1927.474733] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Acquired lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1927.474890] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1927.582268] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1927.609021] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1927.609021] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1927.609021] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1927.609639] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1927.609639] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1927.609639] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1927.609639] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1927.609639] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1927.609860] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1927.609860] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1927.609860] env[62506]: DEBUG nova.virt.hardware [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1927.609860] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d95b84f-238f-47b6-ad20-670c5c364a47 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1927.617265] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6df062ce-f524-4d9f-ac00-34cf21c08761 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1927.913409] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1928.010711] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1928.149952] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1928.220840] env[62506]: DEBUG nova.compute.manager [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Received event network-changed-b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1928.221049] env[62506]: DEBUG nova.compute.manager [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Refreshing instance network info cache due to event network-changed-b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1928.221264] env[62506]: DEBUG oslo_concurrency.lockutils [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] Acquiring lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1928.221406] env[62506]: DEBUG oslo_concurrency.lockutils [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] Acquired lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1928.221564] env[62506]: DEBUG nova.network.neutron [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Refreshing network info cache for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1928.414494] env[62506]: ERROR nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1928.414494] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1928.414494] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1928.414494] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1928.414494] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1928.414494] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1928.414494] env[62506]: ERROR nova.compute.manager raise self.value [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1928.414494] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1928.414494] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1928.414494] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1928.415115] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1928.415115] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1928.415115] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1928.415115] env[62506]: ERROR nova.compute.manager [ 1928.415115] env[62506]: Traceback (most recent call last): [ 1928.415115] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1928.415115] env[62506]: listener.cb(fileno) [ 1928.415115] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1928.415115] env[62506]: result = function(*args, **kwargs) [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1928.415115] env[62506]: return func(*args, **kwargs) [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1928.415115] env[62506]: raise e [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1928.415115] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1928.415115] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1928.415115] env[62506]: with excutils.save_and_reraise_exception(): [ 1928.415115] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1928.415115] env[62506]: self.force_reraise() [ 1928.415115] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1928.415115] env[62506]: raise self.value [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1928.415115] env[62506]: updated_port = self._update_port( [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1928.415115] env[62506]: _ensure_no_port_binding_failure(port) [ 1928.415115] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1928.415115] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1928.415805] env[62506]: nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1928.415805] env[62506]: Removing descriptor: 19 [ 1928.415805] env[62506]: ERROR nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Traceback (most recent call last): [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] yield resources [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self.driver.spawn(context, instance, image_meta, [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1928.415805] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] vm_ref = self.build_virtual_machine(instance, [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] vif_infos = vmwarevif.get_vif_info(self._session, [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] for vif in network_info: [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return self._sync_wrapper(fn, *args, **kwargs) [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self.wait() [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self[:] = self._gt.wait() [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return self._exit_event.wait() [ 1928.416128] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] result = hub.switch() [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return self.greenlet.switch() [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] result = function(*args, **kwargs) [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return func(*args, **kwargs) [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] raise e [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] nwinfo = self.network_api.allocate_for_instance( [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1928.416454] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] created_port_ids = self._update_ports_for_instance( [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] with excutils.save_and_reraise_exception(): [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self.force_reraise() [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] raise self.value [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] updated_port = self._update_port( [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] _ensure_no_port_binding_failure(port) [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1928.416782] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] raise exception.PortBindingFailed(port_id=port['id']) [ 1928.417090] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1928.417090] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] [ 1928.417090] env[62506]: INFO nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Terminating instance [ 1928.609913] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1928.610112] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 1928.610296] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1928.652381] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Releasing lock "refresh_cache-1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1928.652632] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1928.652878] env[62506]: DEBUG nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1928.653086] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1928.690654] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1928.749932] env[62506]: DEBUG nova.network.neutron [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1928.920154] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquiring lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1928.925411] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5da3cfd-c65a-4cf7-aa8e-2631d16c12b9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1928.933430] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28615bab-3c97-4964-b880-a690ba84a27b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1928.942462] env[62506]: DEBUG nova.network.neutron [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1928.969670] env[62506]: DEBUG oslo_concurrency.lockutils [req-9a039ad0-76e6-4c4f-8964-f65fa71d1eb0 req-f9bb58e3-2350-4179-b1d2-8314be7c791e service nova] Releasing lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1928.970579] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c948d87e-cb73-4684-8f72-5744747cc92d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1928.973454] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquired lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1928.973645] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1928.982310] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a13d8dfe-aab7-4c8e-b00a-46adb6032085 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1928.997487] env[62506]: DEBUG nova.compute.provider_tree [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1929.114324] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1929.193621] env[62506]: DEBUG nova.network.neutron [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1929.498151] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1929.503597] env[62506]: DEBUG nova.scheduler.client.report [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1929.623140] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1929.697894] env[62506]: INFO nova.compute.manager [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] [instance: 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265] Took 1.04 seconds to deallocate network for instance. [ 1930.005561] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.536s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1930.006278] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1930.011489] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 27.759s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1930.011489] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1930.011489] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.003s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1930.037566] env[62506]: INFO nova.scheduler.client.report [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Deleted allocations for instance b4bcab92-0007-4952-8d74-c470049edd71 [ 1930.126950] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Releasing lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1930.126950] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1930.126950] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1930.126950] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a456b560-896f-48fc-bf05-d18f9d6d9b31 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1930.135857] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37e88e65-6051-4081-a66a-b2eafb64160d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1930.158385] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 43d09ce7-7946-43d1-8729-d7068c1cdec4 could not be found. [ 1930.158627] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1930.158846] env[62506]: INFO nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1930.159112] env[62506]: DEBUG oslo.service.loopingcall [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1930.159337] env[62506]: DEBUG nova.compute.manager [-] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1930.159432] env[62506]: DEBUG nova.network.neutron [-] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1930.175957] env[62506]: DEBUG nova.network.neutron [-] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1930.321221] env[62506]: DEBUG nova.compute.manager [req-38e755f4-977d-47ba-bb8b-d203ce5e387a req-6e854eb5-77ac-4458-adbd-4bb3a0d50c73 service nova] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Received event network-vif-deleted-b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1930.516670] env[62506]: DEBUG nova.compute.utils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1930.523016] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1930.523016] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1930.548073] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c5f4e4f-5fd5-44eb-85dd-1e5d236b4dfc tempest-ServerDiagnosticsV248Test-1496531621 tempest-ServerDiagnosticsV248Test-1496531621-project-member] Lock "b4bcab92-0007-4952-8d74-c470049edd71" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 32.578s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1930.598094] env[62506]: DEBUG nova.policy [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ccbe5c9869a040728e5929f30a095e27', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ef8974982916474c9120abcde7f3cfae', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1930.678221] env[62506]: DEBUG nova.network.neutron [-] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1930.731473] env[62506]: INFO nova.scheduler.client.report [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Deleted allocations for instance 1d4d567b-c1b0-4bf3-96d1-405c3a1cd265 [ 1930.882639] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2faa0f38-c0b7-4322-a86b-650ba0afcdbd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1930.891924] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-365d40ee-973f-4c77-ab14-09a4945de790 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1930.924440] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28b1c53b-e8cf-4940-ae0d-4091e1b75046 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1930.933324] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b1b8891-516e-444f-aef6-05f660747cbc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1930.952239] env[62506]: DEBUG nova.compute.provider_tree [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1931.024592] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1931.182166] env[62506]: INFO nova.compute.manager [-] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Took 1.02 seconds to deallocate network for instance. [ 1931.183507] env[62506]: DEBUG nova.compute.claims [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1931.183812] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1931.247169] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8f7e9516-3891-4daf-9540-8f2d40cbc582 tempest-ServerDiagnosticsTest-161353466 tempest-ServerDiagnosticsTest-161353466-project-member] Lock "1d4d567b-c1b0-4bf3-96d1-405c3a1cd265" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 103.140s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1931.252974] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Successfully created port: be9686dd-6786-49af-9994-03f259ddac8b {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1931.458023] env[62506]: DEBUG nova.scheduler.client.report [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1931.758213] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1931.965427] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.954s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1931.967580] env[62506]: ERROR nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Traceback (most recent call last): [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self.driver.spawn(context, instance, image_meta, [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] vm_ref = self.build_virtual_machine(instance, [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] vif_infos = vmwarevif.get_vif_info(self._session, [ 1931.967580] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] for vif in network_info: [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return self._sync_wrapper(fn, *args, **kwargs) [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self.wait() [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self[:] = self._gt.wait() [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return self._exit_event.wait() [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] result = hub.switch() [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1931.967965] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return self.greenlet.switch() [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] result = function(*args, **kwargs) [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] return func(*args, **kwargs) [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] raise e [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] nwinfo = self.network_api.allocate_for_instance( [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] created_port_ids = self._update_ports_for_instance( [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] with excutils.save_and_reraise_exception(): [ 1931.968301] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] self.force_reraise() [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] raise self.value [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] updated_port = self._update_port( [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] _ensure_no_port_binding_failure(port) [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] raise exception.PortBindingFailed(port_id=port['id']) [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] nova.exception.PortBindingFailed: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. [ 1931.968674] env[62506]: ERROR nova.compute.manager [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] [ 1931.968990] env[62506]: DEBUG nova.compute.utils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1931.972186] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.863s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1931.973689] env[62506]: INFO nova.compute.claims [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1931.977810] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Build of instance 7e185b7e-8a95-4931-96a8-1c27516f767d was re-scheduled: Binding failed for port 7549bedb-b15e-4c50-b70a-a32f5a98e486, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1931.978313] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1931.978541] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1931.978708] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquired lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1931.978839] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1932.038086] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1932.064894] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1932.065228] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1932.065228] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1932.066594] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1932.066594] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1932.066594] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1932.067409] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1932.067409] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1932.067409] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1932.067409] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1932.067409] env[62506]: DEBUG nova.virt.hardware [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1932.069567] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26ee83cd-860d-4ec9-b39a-69bc2d03d9c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1932.078423] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b125567-4ade-4a0a-8449-7000c36ba7be {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1932.286232] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1932.520126] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1932.729201] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1933.231420] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Releasing lock "refresh_cache-7e185b7e-8a95-4931-96a8-1c27516f767d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1933.231709] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1933.231898] env[62506]: DEBUG nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1933.232079] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1933.270672] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1933.333250] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e925f0e-11fe-4d39-889a-322211347c8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1933.342145] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d747d83-35d9-43f6-b173-0c41d241c55f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1933.379202] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6eb975f1-e9f5-48df-9a1b-8fa99148cd7e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1933.384643] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9d95303-54bc-429e-9779-1360222b4a2e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1933.400242] env[62506]: DEBUG nova.compute.provider_tree [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1933.459568] env[62506]: DEBUG nova.compute.manager [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Received event network-changed-be9686dd-6786-49af-9994-03f259ddac8b {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1933.459759] env[62506]: DEBUG nova.compute.manager [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Refreshing instance network info cache due to event network-changed-be9686dd-6786-49af-9994-03f259ddac8b. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1933.459975] env[62506]: DEBUG oslo_concurrency.lockutils [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] Acquiring lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1933.460123] env[62506]: DEBUG oslo_concurrency.lockutils [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] Acquired lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1933.460276] env[62506]: DEBUG nova.network.neutron [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Refreshing network info cache for port be9686dd-6786-49af-9994-03f259ddac8b {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1933.628423] env[62506]: ERROR nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1933.628423] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1933.628423] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1933.628423] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1933.628423] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1933.628423] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1933.628423] env[62506]: ERROR nova.compute.manager raise self.value [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1933.628423] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1933.628423] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1933.628423] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1933.628916] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1933.628916] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1933.628916] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1933.628916] env[62506]: ERROR nova.compute.manager [ 1933.628916] env[62506]: Traceback (most recent call last): [ 1933.628916] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1933.628916] env[62506]: listener.cb(fileno) [ 1933.628916] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1933.628916] env[62506]: result = function(*args, **kwargs) [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1933.628916] env[62506]: return func(*args, **kwargs) [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1933.628916] env[62506]: raise e [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1933.628916] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1933.628916] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1933.628916] env[62506]: with excutils.save_and_reraise_exception(): [ 1933.628916] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1933.628916] env[62506]: self.force_reraise() [ 1933.628916] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1933.628916] env[62506]: raise self.value [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1933.628916] env[62506]: updated_port = self._update_port( [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1933.628916] env[62506]: _ensure_no_port_binding_failure(port) [ 1933.628916] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1933.628916] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1933.629702] env[62506]: nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1933.629702] env[62506]: Removing descriptor: 19 [ 1933.629702] env[62506]: ERROR nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Traceback (most recent call last): [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] yield resources [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self.driver.spawn(context, instance, image_meta, [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1933.629702] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] vm_ref = self.build_virtual_machine(instance, [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] vif_infos = vmwarevif.get_vif_info(self._session, [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] for vif in network_info: [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return self._sync_wrapper(fn, *args, **kwargs) [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self.wait() [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self[:] = self._gt.wait() [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return self._exit_event.wait() [ 1933.629997] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] result = hub.switch() [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return self.greenlet.switch() [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] result = function(*args, **kwargs) [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return func(*args, **kwargs) [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] raise e [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] nwinfo = self.network_api.allocate_for_instance( [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1933.630352] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] created_port_ids = self._update_ports_for_instance( [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] with excutils.save_and_reraise_exception(): [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self.force_reraise() [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] raise self.value [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] updated_port = self._update_port( [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] _ensure_no_port_binding_failure(port) [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1933.630724] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] raise exception.PortBindingFailed(port_id=port['id']) [ 1933.631061] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1933.631061] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] [ 1933.631061] env[62506]: INFO nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Terminating instance [ 1933.771903] env[62506]: DEBUG nova.network.neutron [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1933.907917] env[62506]: DEBUG nova.scheduler.client.report [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1933.982474] env[62506]: DEBUG nova.network.neutron [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1934.124173] env[62506]: DEBUG nova.network.neutron [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1934.139738] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquiring lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1934.277153] env[62506]: INFO nova.compute.manager [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 7e185b7e-8a95-4931-96a8-1c27516f767d] Took 1.04 seconds to deallocate network for instance. [ 1934.415143] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.447s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1934.415688] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1934.420616] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.259s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1934.421991] env[62506]: INFO nova.compute.claims [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1934.627794] env[62506]: DEBUG oslo_concurrency.lockutils [req-5b053c5b-5a39-447c-9f02-06fdb77b2d3a req-6a0ee4c0-40a7-4439-8bf9-147b336df8ee service nova] Releasing lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1934.628863] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquired lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1934.628863] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1934.924572] env[62506]: DEBUG nova.compute.utils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1934.925983] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1934.927074] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1935.003791] env[62506]: DEBUG nova.policy [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2afe9bed682d466da44dfd83b8fb3a50', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0314fb364e7743a69e28b348a91d111b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1935.021636] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "3e30fb9a-092b-4886-b496-9a54cd4477d2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1935.021882] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "3e30fb9a-092b-4886-b496-9a54cd4477d2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1935.162116] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1935.320034] env[62506]: INFO nova.scheduler.client.report [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Deleted allocations for instance 7e185b7e-8a95-4931-96a8-1c27516f767d [ 1935.335732] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1935.434999] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1935.522676] env[62506]: DEBUG nova.compute.manager [req-129a71f3-d49a-4485-a9bb-4cbd18200be0 req-09915d2b-6f35-4bcc-bf74-99c3295ab609 service nova] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Received event network-vif-deleted-be9686dd-6786-49af-9994-03f259ddac8b {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1935.718260] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Successfully created port: 485593e6-f77a-42f4-ae45-94d9e51d1679 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1935.825941] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29ef3044-009c-4f1e-b5ff-ee4445f59330 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1935.830898] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ef4e357a-a633-42b2-a327-1906008fdfda tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "7e185b7e-8a95-4931-96a8-1c27516f767d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 101.366s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1935.834828] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16882415-9bac-43d4-ac8d-3ef3c946fa92 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1935.838293] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Releasing lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1935.838699] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1935.838890] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1935.839166] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8fa9a8fa-c44e-4c2b-b2f2-dc5bf8395357 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1935.871240] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1698a47-e047-4f67-9fa7-a33b7962fe2c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1935.876684] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eeefe609-0d1b-48a6-a407-617d5bf9d8e3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1935.893537] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9f3030c-5cc9-45bc-8888-46df4a75dce9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1935.901711] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9 could not be found. [ 1935.901957] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1935.902470] env[62506]: INFO nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1935.902470] env[62506]: DEBUG oslo.service.loopingcall [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1935.902922] env[62506]: DEBUG nova.compute.manager [-] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1935.903033] env[62506]: DEBUG nova.network.neutron [-] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1935.912465] env[62506]: DEBUG nova.compute.provider_tree [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1935.940068] env[62506]: DEBUG nova.network.neutron [-] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1936.335932] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1936.419010] env[62506]: DEBUG nova.scheduler.client.report [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1936.442383] env[62506]: DEBUG nova.network.neutron [-] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1936.447197] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1936.478604] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1936.479159] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1936.479159] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1936.479324] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1936.479371] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1936.479522] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1936.479731] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1936.479894] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1936.480136] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1936.480353] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1936.480538] env[62506]: DEBUG nova.virt.hardware [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1936.482131] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df2c38c0-7271-4b0e-b08c-4fc0a8d79900 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1936.492593] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7cbdc90-f64f-40b2-b0e2-4155ed78e46e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1936.862253] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1936.924348] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.504s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1936.924860] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1936.928047] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.027s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1936.946474] env[62506]: INFO nova.compute.manager [-] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Took 1.04 seconds to deallocate network for instance. [ 1936.948172] env[62506]: DEBUG nova.compute.claims [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1936.948249] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1937.155026] env[62506]: ERROR nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1937.155026] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1937.155026] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1937.155026] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1937.155026] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1937.155026] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1937.155026] env[62506]: ERROR nova.compute.manager raise self.value [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1937.155026] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1937.155026] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1937.155026] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1937.155450] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1937.155450] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1937.155450] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1937.155450] env[62506]: ERROR nova.compute.manager [ 1937.155450] env[62506]: Traceback (most recent call last): [ 1937.155450] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1937.155450] env[62506]: listener.cb(fileno) [ 1937.155450] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1937.155450] env[62506]: result = function(*args, **kwargs) [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1937.155450] env[62506]: return func(*args, **kwargs) [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1937.155450] env[62506]: raise e [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1937.155450] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1937.155450] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1937.155450] env[62506]: with excutils.save_and_reraise_exception(): [ 1937.155450] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1937.155450] env[62506]: self.force_reraise() [ 1937.155450] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1937.155450] env[62506]: raise self.value [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1937.155450] env[62506]: updated_port = self._update_port( [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1937.155450] env[62506]: _ensure_no_port_binding_failure(port) [ 1937.155450] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1937.155450] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1937.156165] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1937.156165] env[62506]: Removing descriptor: 19 [ 1937.156165] env[62506]: ERROR nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Traceback (most recent call last): [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] yield resources [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self.driver.spawn(context, instance, image_meta, [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1937.156165] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] vm_ref = self.build_virtual_machine(instance, [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] vif_infos = vmwarevif.get_vif_info(self._session, [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] for vif in network_info: [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return self._sync_wrapper(fn, *args, **kwargs) [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self.wait() [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self[:] = self._gt.wait() [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return self._exit_event.wait() [ 1937.156468] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] result = hub.switch() [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return self.greenlet.switch() [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] result = function(*args, **kwargs) [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return func(*args, **kwargs) [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] raise e [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] nwinfo = self.network_api.allocate_for_instance( [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1937.156821] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] created_port_ids = self._update_ports_for_instance( [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] with excutils.save_and_reraise_exception(): [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self.force_reraise() [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] raise self.value [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] updated_port = self._update_port( [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] _ensure_no_port_binding_failure(port) [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1937.157158] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] raise exception.PortBindingFailed(port_id=port['id']) [ 1937.157480] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1937.157480] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] [ 1937.157480] env[62506]: INFO nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Terminating instance [ 1937.431295] env[62506]: DEBUG nova.compute.utils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1937.433423] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1937.433656] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1937.522320] env[62506]: DEBUG nova.policy [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2afe9bed682d466da44dfd83b8fb3a50', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0314fb364e7743a69e28b348a91d111b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1937.588134] env[62506]: DEBUG nova.compute.manager [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Received event network-changed-485593e6-f77a-42f4-ae45-94d9e51d1679 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1937.588227] env[62506]: DEBUG nova.compute.manager [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Refreshing instance network info cache due to event network-changed-485593e6-f77a-42f4-ae45-94d9e51d1679. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1937.588465] env[62506]: DEBUG oslo_concurrency.lockutils [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] Acquiring lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1937.588569] env[62506]: DEBUG oslo_concurrency.lockutils [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] Acquired lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1937.588737] env[62506]: DEBUG nova.network.neutron [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Refreshing network info cache for port 485593e6-f77a-42f4-ae45-94d9e51d1679 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1937.664724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1937.759239] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a736cc4-0939-45ae-9469-46421ad691f3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1937.767975] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9066543d-516e-475c-bae3-3061d18943c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1937.801073] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-110d5bc0-ac56-49f9-bbb6-a0231bb44a79 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1937.808959] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9010f4c-5e8d-4ab5-914c-5bd4bbb981dc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1937.823454] env[62506]: DEBUG nova.compute.provider_tree [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1937.872060] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Successfully created port: bdaecc2f-5bee-4b03-b42c-ea028e363d10 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1937.937087] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1938.120759] env[62506]: DEBUG nova.network.neutron [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1938.249594] env[62506]: DEBUG nova.network.neutron [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1938.326802] env[62506]: DEBUG nova.scheduler.client.report [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1938.752862] env[62506]: DEBUG oslo_concurrency.lockutils [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] Releasing lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1938.753191] env[62506]: DEBUG nova.compute.manager [req-31fc250b-a27e-458b-9983-efed543f5f9f req-39271f49-d7a1-4cb3-b5fd-0e627dccd21c service nova] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Received event network-vif-deleted-485593e6-f77a-42f4-ae45-94d9e51d1679 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1938.753559] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquired lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1938.753734] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1938.832394] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.904s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1938.832714] env[62506]: ERROR nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Traceback (most recent call last): [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self.driver.spawn(context, instance, image_meta, [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] vm_ref = self.build_virtual_machine(instance, [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] vif_infos = vmwarevif.get_vif_info(self._session, [ 1938.832714] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] for vif in network_info: [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return self._sync_wrapper(fn, *args, **kwargs) [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self.wait() [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self[:] = self._gt.wait() [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return self._exit_event.wait() [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] result = hub.switch() [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1938.834857] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return self.greenlet.switch() [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] result = function(*args, **kwargs) [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] return func(*args, **kwargs) [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] raise e [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] nwinfo = self.network_api.allocate_for_instance( [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] created_port_ids = self._update_ports_for_instance( [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] with excutils.save_and_reraise_exception(): [ 1938.835263] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] self.force_reraise() [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] raise self.value [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] updated_port = self._update_port( [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] _ensure_no_port_binding_failure(port) [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] raise exception.PortBindingFailed(port_id=port['id']) [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] nova.exception.PortBindingFailed: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. [ 1938.835643] env[62506]: ERROR nova.compute.manager [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] [ 1938.835978] env[62506]: DEBUG nova.compute.utils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1938.835978] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 25.438s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1938.837800] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Build of instance e0de75e0-ca8e-4525-ae7f-ed13fcd07338 was re-scheduled: Binding failed for port d015a181-70f8-4b40-bef7-e581cbfb668d, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1938.838620] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1938.838620] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquiring lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1938.838620] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Acquired lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1938.838842] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1938.945884] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1938.982681] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1938.982919] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1938.983083] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1938.983266] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1938.983408] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1938.985049] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1938.985294] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1938.985567] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1938.985567] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1938.985832] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1938.987167] env[62506]: DEBUG nova.virt.hardware [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1938.987167] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8441c763-5218-47e4-9204-e75aafd9fad9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1938.996977] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9928673a-fa0a-4144-a131-17685e5ae5a0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.063384] env[62506]: ERROR nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1939.063384] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1939.063384] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1939.063384] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1939.063384] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1939.063384] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1939.063384] env[62506]: ERROR nova.compute.manager raise self.value [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1939.063384] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1939.063384] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1939.063384] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1939.063919] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1939.063919] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1939.063919] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1939.063919] env[62506]: ERROR nova.compute.manager [ 1939.063919] env[62506]: Traceback (most recent call last): [ 1939.063919] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1939.063919] env[62506]: listener.cb(fileno) [ 1939.063919] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1939.063919] env[62506]: result = function(*args, **kwargs) [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1939.063919] env[62506]: return func(*args, **kwargs) [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1939.063919] env[62506]: raise e [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1939.063919] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1939.063919] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1939.063919] env[62506]: with excutils.save_and_reraise_exception(): [ 1939.063919] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1939.063919] env[62506]: self.force_reraise() [ 1939.063919] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1939.063919] env[62506]: raise self.value [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1939.063919] env[62506]: updated_port = self._update_port( [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1939.063919] env[62506]: _ensure_no_port_binding_failure(port) [ 1939.063919] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1939.063919] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1939.064771] env[62506]: nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1939.064771] env[62506]: Removing descriptor: 19 [ 1939.064771] env[62506]: ERROR nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Traceback (most recent call last): [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] yield resources [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self.driver.spawn(context, instance, image_meta, [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1939.064771] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] vm_ref = self.build_virtual_machine(instance, [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] vif_infos = vmwarevif.get_vif_info(self._session, [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] for vif in network_info: [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return self._sync_wrapper(fn, *args, **kwargs) [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self.wait() [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self[:] = self._gt.wait() [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return self._exit_event.wait() [ 1939.065121] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] result = hub.switch() [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return self.greenlet.switch() [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] result = function(*args, **kwargs) [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return func(*args, **kwargs) [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] raise e [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] nwinfo = self.network_api.allocate_for_instance( [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1939.065474] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] created_port_ids = self._update_ports_for_instance( [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] with excutils.save_and_reraise_exception(): [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self.force_reraise() [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] raise self.value [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] updated_port = self._update_port( [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] _ensure_no_port_binding_failure(port) [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1939.065852] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] raise exception.PortBindingFailed(port_id=port['id']) [ 1939.066211] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1939.066211] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] [ 1939.066211] env[62506]: INFO nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Terminating instance [ 1939.281270] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1939.347852] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1939.363371] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1939.458346] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1939.571196] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1939.571389] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquired lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1939.571567] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1939.655832] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-451e0a77-7fc2-457a-a1e1-eaf327afc2f4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.660709] env[62506]: DEBUG nova.compute.manager [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Received event network-changed-bdaecc2f-5bee-4b03-b42c-ea028e363d10 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1939.661810] env[62506]: DEBUG nova.compute.manager [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Refreshing instance network info cache due to event network-changed-bdaecc2f-5bee-4b03-b42c-ea028e363d10. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1939.661810] env[62506]: DEBUG oslo_concurrency.lockutils [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] Acquiring lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1939.667014] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91c678f3-3a48-4c21-a8fb-f77e9f8319bc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.704234] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6cf0962-e96e-4458-88bb-dffa51a1e773 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.711824] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5aea14a3-b909-4213-8939-8954d1e17ece {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.726069] env[62506]: DEBUG nova.compute.provider_tree [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1939.855671] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Releasing lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1939.855671] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1939.855671] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1939.855961] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-dee8582a-eb38-4942-a718-8d0194569bb3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.866315] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daa169b1-fe57-4145-9dbc-8fbfc5f2fd2e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.893753] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 01ad7fbe-7ea0-425b-ba63-28537cb158b1 could not be found. [ 1939.893994] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1939.894330] env[62506]: INFO nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1939.895526] env[62506]: DEBUG oslo.service.loopingcall [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1939.896198] env[62506]: DEBUG nova.compute.manager [-] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1939.896198] env[62506]: DEBUG nova.network.neutron [-] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1939.933503] env[62506]: DEBUG nova.network.neutron [-] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1939.963724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Releasing lock "refresh_cache-e0de75e0-ca8e-4525-ae7f-ed13fcd07338" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1939.963724] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1939.963724] env[62506]: DEBUG nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1939.963724] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1939.986464] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1940.100960] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1940.183179] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1940.231263] env[62506]: DEBUG nova.scheduler.client.report [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1940.436859] env[62506]: DEBUG nova.network.neutron [-] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1940.489796] env[62506]: DEBUG nova.network.neutron [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1940.686967] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Releasing lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1940.686967] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1940.686967] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1940.687328] env[62506]: DEBUG oslo_concurrency.lockutils [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] Acquired lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1940.687385] env[62506]: DEBUG nova.network.neutron [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Refreshing network info cache for port bdaecc2f-5bee-4b03-b42c-ea028e363d10 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1940.688610] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7f9366df-226d-4748-8adf-ad4495a5423e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1940.700182] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75054ddd-191b-462d-9282-f980c6feac01 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1940.724445] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 854c955f-b327-4c25-87bd-3ad68dbff024 could not be found. [ 1940.724689] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1940.724865] env[62506]: INFO nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1940.725113] env[62506]: DEBUG oslo.service.loopingcall [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1940.725583] env[62506]: DEBUG nova.compute.manager [-] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1940.725681] env[62506]: DEBUG nova.network.neutron [-] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1940.737257] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.903s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1940.738027] env[62506]: ERROR nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Traceback (most recent call last): [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self.driver.spawn(context, instance, image_meta, [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] vm_ref = self.build_virtual_machine(instance, [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] vif_infos = vmwarevif.get_vif_info(self._session, [ 1940.738027] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] for vif in network_info: [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return self._sync_wrapper(fn, *args, **kwargs) [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self.wait() [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self[:] = self._gt.wait() [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return self._exit_event.wait() [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] result = hub.switch() [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1940.738410] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return self.greenlet.switch() [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] result = function(*args, **kwargs) [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] return func(*args, **kwargs) [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] raise e [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] nwinfo = self.network_api.allocate_for_instance( [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] created_port_ids = self._update_ports_for_instance( [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] with excutils.save_and_reraise_exception(): [ 1940.738753] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] self.force_reraise() [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] raise self.value [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] updated_port = self._update_port( [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] _ensure_no_port_binding_failure(port) [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] raise exception.PortBindingFailed(port_id=port['id']) [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] nova.exception.PortBindingFailed: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. [ 1940.739114] env[62506]: ERROR nova.compute.manager [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] [ 1940.739418] env[62506]: DEBUG nova.compute.utils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1940.739784] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.602s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1940.741964] env[62506]: INFO nova.compute.claims [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1940.745492] env[62506]: DEBUG nova.network.neutron [-] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1940.746990] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Build of instance 03e1ebe5-7a02-430e-b38c-d215e85d59ae was re-scheduled: Binding failed for port 121430c8-695c-4a4f-bbeb-62fec4457c07, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1940.747454] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1940.747802] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquiring lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1940.747965] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Acquired lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1940.748139] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1940.939792] env[62506]: INFO nova.compute.manager [-] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Took 1.04 seconds to deallocate network for instance. [ 1940.943882] env[62506]: DEBUG nova.compute.claims [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1940.944085] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1940.993185] env[62506]: INFO nova.compute.manager [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] [instance: e0de75e0-ca8e-4525-ae7f-ed13fcd07338] Took 1.03 seconds to deallocate network for instance. [ 1941.209011] env[62506]: DEBUG nova.network.neutron [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1941.247676] env[62506]: DEBUG nova.network.neutron [-] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1941.266524] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1941.331926] env[62506]: DEBUG nova.network.neutron [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1941.417412] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1941.752968] env[62506]: INFO nova.compute.manager [-] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Took 1.03 seconds to deallocate network for instance. [ 1941.755248] env[62506]: DEBUG nova.compute.claims [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1941.755431] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1941.834704] env[62506]: DEBUG oslo_concurrency.lockutils [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] Releasing lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1941.834951] env[62506]: DEBUG nova.compute.manager [req-33dc0a7a-960f-4539-9017-19e7636989af req-9295c406-5816-4836-8eac-d8669d1d54c8 service nova] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Received event network-vif-deleted-bdaecc2f-5bee-4b03-b42c-ea028e363d10 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1941.920209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Releasing lock "refresh_cache-03e1ebe5-7a02-430e-b38c-d215e85d59ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1941.920450] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1941.920688] env[62506]: DEBUG nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1941.920888] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1941.936225] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1942.016432] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a221cee5-4516-4d71-b9a9-86fad4902fc4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1942.019726] env[62506]: INFO nova.scheduler.client.report [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Deleted allocations for instance e0de75e0-ca8e-4525-ae7f-ed13fcd07338 [ 1942.030145] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e41070f-fa46-4ed3-ae05-9f18431f57ff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1942.061760] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-874e7afb-b140-4be8-a55e-6b9ec9a939f3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1942.072021] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-444f6da9-eb5c-4fd5-a549-d90ce717f733 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1942.084776] env[62506]: DEBUG nova.compute.provider_tree [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1942.437993] env[62506]: DEBUG nova.network.neutron [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1942.528471] env[62506]: DEBUG oslo_concurrency.lockutils [None req-01481f5b-5897-4c83-8cbc-7f43862f1a28 tempest-ImagesOneServerNegativeTestJSON-1299567646 tempest-ImagesOneServerNegativeTestJSON-1299567646-project-member] Lock "e0de75e0-ca8e-4525-ae7f-ed13fcd07338" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 106.797s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1942.588173] env[62506]: DEBUG nova.scheduler.client.report [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1942.941761] env[62506]: INFO nova.compute.manager [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] [instance: 03e1ebe5-7a02-430e-b38c-d215e85d59ae] Took 1.02 seconds to deallocate network for instance. [ 1943.031726] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1943.094636] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.355s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1943.097013] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1943.098234] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.185s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1943.099724] env[62506]: INFO nova.compute.claims [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1943.554323] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1943.604424] env[62506]: DEBUG nova.compute.utils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1943.608479] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1943.608479] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1943.677404] env[62506]: DEBUG nova.policy [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b37c0529800a43eca96641d95d60394f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '68001a3c57354f5ab7b9010a90ac0c92', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1943.986019] env[62506]: INFO nova.scheduler.client.report [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Deleted allocations for instance 03e1ebe5-7a02-430e-b38c-d215e85d59ae [ 1944.109402] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1944.327445] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Successfully created port: d52c1b94-5eb1-441a-b6e6-0a2f53e2f782 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1944.423980] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85fc7f32-fcae-402e-8157-9fd0790b3ad6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1944.435352] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7056410-db2e-4fe1-8fe3-8b0447d4e846 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1944.471599] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7af76255-5e6c-4dfa-9f3f-310dc606c97f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1944.481432] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-504d33d7-73d0-4046-8522-c4531e02397d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1944.498510] env[62506]: DEBUG oslo_concurrency.lockutils [None req-295216e4-c575-4dbc-8e3d-03959b23bce1 tempest-ServerRescueNegativeTestJSON-615250978 tempest-ServerRescueNegativeTestJSON-615250978-project-member] Lock "03e1ebe5-7a02-430e-b38c-d215e85d59ae" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 106.094s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1944.498998] env[62506]: DEBUG nova.compute.provider_tree [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1945.005178] env[62506]: DEBUG nova.scheduler.client.report [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1945.007356] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1945.119845] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1945.145084] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1945.145351] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1945.145497] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1945.145787] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1945.145840] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1945.146019] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1945.146236] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1945.146398] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1945.146561] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1945.146803] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1945.146903] env[62506]: DEBUG nova.virt.hardware [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1945.147802] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65646284-ebcb-4737-b955-2a3721ebf40c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1945.157150] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7845946-1543-4005-9f43-af9e7628c2b8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1945.512044] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.414s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1945.512605] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1945.525025] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 16.407s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1945.525025] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.002s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1945.525025] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1945.525025] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.340s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1945.529465] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-854bc0fe-5a63-4743-af75-e0c511c73dd0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1945.538133] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d2b05b8-aa21-42c9-af6c-61ef26f1cf25 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1945.558944] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1945.561884] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-592ee558-bc0b-4ff2-94be-cf3eea16ef1f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1945.568856] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0106947-b138-49d4-947d-11cd2aebeb8a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1945.600375] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181223MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1945.600541] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1945.898586] env[62506]: DEBUG nova.compute.manager [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Received event network-changed-d52c1b94-5eb1-441a-b6e6-0a2f53e2f782 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1945.898817] env[62506]: DEBUG nova.compute.manager [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Refreshing instance network info cache due to event network-changed-d52c1b94-5eb1-441a-b6e6-0a2f53e2f782. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1945.899677] env[62506]: DEBUG oslo_concurrency.lockutils [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] Acquiring lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1945.899677] env[62506]: DEBUG oslo_concurrency.lockutils [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] Acquired lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1945.899677] env[62506]: DEBUG nova.network.neutron [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Refreshing network info cache for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1946.031206] env[62506]: DEBUG nova.compute.utils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1946.033799] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1946.034016] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1946.126224] env[62506]: DEBUG nova.policy [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '148df6b2aa2d404ca00b81b11a5303d0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'b18f030e5a5347bea027ba79d4e3d1d2', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1946.275167] env[62506]: ERROR nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1946.275167] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1946.275167] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1946.275167] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1946.275167] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1946.275167] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1946.275167] env[62506]: ERROR nova.compute.manager raise self.value [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1946.275167] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1946.275167] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1946.275167] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1946.275587] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1946.275587] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1946.275587] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1946.275587] env[62506]: ERROR nova.compute.manager [ 1946.275587] env[62506]: Traceback (most recent call last): [ 1946.275587] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1946.275587] env[62506]: listener.cb(fileno) [ 1946.275587] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1946.275587] env[62506]: result = function(*args, **kwargs) [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1946.275587] env[62506]: return func(*args, **kwargs) [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1946.275587] env[62506]: raise e [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1946.275587] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1946.275587] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1946.275587] env[62506]: with excutils.save_and_reraise_exception(): [ 1946.275587] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1946.275587] env[62506]: self.force_reraise() [ 1946.275587] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1946.275587] env[62506]: raise self.value [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1946.275587] env[62506]: updated_port = self._update_port( [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1946.275587] env[62506]: _ensure_no_port_binding_failure(port) [ 1946.275587] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1946.275587] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1946.276298] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1946.276298] env[62506]: Removing descriptor: 19 [ 1946.276298] env[62506]: ERROR nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Traceback (most recent call last): [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] yield resources [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self.driver.spawn(context, instance, image_meta, [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1946.276298] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] vm_ref = self.build_virtual_machine(instance, [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] vif_infos = vmwarevif.get_vif_info(self._session, [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] for vif in network_info: [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return self._sync_wrapper(fn, *args, **kwargs) [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self.wait() [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self[:] = self._gt.wait() [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return self._exit_event.wait() [ 1946.276643] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] result = hub.switch() [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return self.greenlet.switch() [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] result = function(*args, **kwargs) [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return func(*args, **kwargs) [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] raise e [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] nwinfo = self.network_api.allocate_for_instance( [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1946.277031] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] created_port_ids = self._update_ports_for_instance( [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] with excutils.save_and_reraise_exception(): [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self.force_reraise() [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] raise self.value [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] updated_port = self._update_port( [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] _ensure_no_port_binding_failure(port) [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1946.277377] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] raise exception.PortBindingFailed(port_id=port['id']) [ 1946.277748] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1946.277748] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] [ 1946.277748] env[62506]: INFO nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Terminating instance [ 1946.344037] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-140a1e8b-5934-48b5-85e2-a7087712f4a7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1946.352707] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9377b6bb-3bd3-4f07-9636-67cde9a6ae09 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1946.389747] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac933e7c-7cbf-47a8-8ecb-1a330b938c86 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1946.398172] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f91f643d-4f61-428c-a184-e121edb7848c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1946.418122] env[62506]: DEBUG nova.compute.provider_tree [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1946.436999] env[62506]: DEBUG nova.network.neutron [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1946.535207] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1946.680315] env[62506]: DEBUG nova.network.neutron [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1946.784723] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1946.892721] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Successfully created port: 60828df4-606a-4cb9-aa71-e54ffe880915 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1946.920897] env[62506]: DEBUG nova.scheduler.client.report [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1947.184161] env[62506]: DEBUG oslo_concurrency.lockutils [req-4431388d-49e8-4b78-aa83-05b113370ea3 req-a65874e6-3d0f-4c61-9a13-b9fefbb82e1e service nova] Releasing lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1947.184912] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquired lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1947.184912] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1947.429923] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.906s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1947.430484] env[62506]: ERROR nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Traceback (most recent call last): [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self.driver.spawn(context, instance, image_meta, [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] vm_ref = self.build_virtual_machine(instance, [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] vif_infos = vmwarevif.get_vif_info(self._session, [ 1947.430484] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] for vif in network_info: [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return self._sync_wrapper(fn, *args, **kwargs) [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self.wait() [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self[:] = self._gt.wait() [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return self._exit_event.wait() [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] result = hub.switch() [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1947.431054] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return self.greenlet.switch() [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] result = function(*args, **kwargs) [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] return func(*args, **kwargs) [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] raise e [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] nwinfo = self.network_api.allocate_for_instance( [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] created_port_ids = self._update_ports_for_instance( [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] with excutils.save_and_reraise_exception(): [ 1947.431749] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] self.force_reraise() [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] raise self.value [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] updated_port = self._update_port( [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] _ensure_no_port_binding_failure(port) [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] raise exception.PortBindingFailed(port_id=port['id']) [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] nova.exception.PortBindingFailed: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. [ 1947.432355] env[62506]: ERROR nova.compute.manager [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] [ 1947.433423] env[62506]: DEBUG nova.compute.utils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1947.433423] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Build of instance 43d09ce7-7946-43d1-8729-d7068c1cdec4 was re-scheduled: Binding failed for port b2d00183-62d9-41f6-8fe7-9ff8fc6c3be1, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1947.433862] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1947.433916] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquiring lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1947.434117] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Acquired lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1947.434275] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1947.435884] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.152s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1947.437448] env[62506]: INFO nova.compute.claims [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1947.543453] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1947.581287] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1947.581545] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1947.581697] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1947.582490] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1947.582778] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1947.582997] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1947.583474] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1947.584349] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1947.584349] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1947.584485] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1947.584713] env[62506]: DEBUG nova.virt.hardware [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1947.586307] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-690957a1-7ba3-4f68-a8d3-95bec14b0fd7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1947.600623] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-232ddcaf-55b4-4757-b7f7-214aa3afd67f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1947.728905] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1947.941035] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1947.960033] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1947.994031] env[62506]: DEBUG nova.compute.manager [req-7b9c8b65-e965-4631-af69-3b7b5d964f65 req-d830654b-8186-41bd-b1b7-90bf5b42b9b6 service nova] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Received event network-vif-deleted-d52c1b94-5eb1-441a-b6e6-0a2f53e2f782 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1948.046175] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1948.445944] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Releasing lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1948.446370] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1948.446817] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1948.449705] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-794e0ef6-df8a-4b1d-a4a0-98312d8ca013 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1948.460364] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dedbadf1-f9d5-4d1e-a1fb-3322c36ae12f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1948.491015] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6cdba5b2-af96-4766-9a97-3a758983c49b could not be found. [ 1948.491373] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1948.491373] env[62506]: INFO nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1948.491754] env[62506]: DEBUG oslo.service.loopingcall [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1948.494939] env[62506]: DEBUG nova.compute.manager [-] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1948.495063] env[62506]: DEBUG nova.network.neutron [-] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1948.516865] env[62506]: DEBUG nova.network.neutron [-] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1948.548829] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Releasing lock "refresh_cache-43d09ce7-7946-43d1-8729-d7068c1cdec4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1948.549568] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1948.549568] env[62506]: DEBUG nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1948.549734] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1948.570308] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1948.835711] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fc5d3f1-0338-44ae-8fa2-3b35bcf0fe26 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1948.845142] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6326bcc6-e6e7-4828-86f3-f5f3b81bcc44 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1948.880099] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6846b4a-2743-49b3-a191-7b867dcde96d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1948.889115] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c75bfb73-be74-4fd0-9866-0baa1cb818ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1948.905304] env[62506]: DEBUG nova.compute.provider_tree [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1949.002184] env[62506]: ERROR nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1949.002184] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1949.002184] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1949.002184] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1949.002184] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1949.002184] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1949.002184] env[62506]: ERROR nova.compute.manager raise self.value [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1949.002184] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1949.002184] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1949.002184] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1949.002643] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1949.002643] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1949.002643] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1949.002643] env[62506]: ERROR nova.compute.manager [ 1949.002643] env[62506]: Traceback (most recent call last): [ 1949.002643] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1949.002643] env[62506]: listener.cb(fileno) [ 1949.002643] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1949.002643] env[62506]: result = function(*args, **kwargs) [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1949.002643] env[62506]: return func(*args, **kwargs) [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1949.002643] env[62506]: raise e [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1949.002643] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1949.002643] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1949.002643] env[62506]: with excutils.save_and_reraise_exception(): [ 1949.002643] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1949.002643] env[62506]: self.force_reraise() [ 1949.002643] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1949.002643] env[62506]: raise self.value [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1949.002643] env[62506]: updated_port = self._update_port( [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1949.002643] env[62506]: _ensure_no_port_binding_failure(port) [ 1949.002643] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1949.002643] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1949.003521] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1949.003521] env[62506]: Removing descriptor: 15 [ 1949.003521] env[62506]: ERROR nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Traceback (most recent call last): [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] yield resources [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self.driver.spawn(context, instance, image_meta, [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1949.003521] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] vm_ref = self.build_virtual_machine(instance, [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] vif_infos = vmwarevif.get_vif_info(self._session, [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] for vif in network_info: [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return self._sync_wrapper(fn, *args, **kwargs) [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self.wait() [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self[:] = self._gt.wait() [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return self._exit_event.wait() [ 1949.003888] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] result = hub.switch() [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return self.greenlet.switch() [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] result = function(*args, **kwargs) [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return func(*args, **kwargs) [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] raise e [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] nwinfo = self.network_api.allocate_for_instance( [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1949.004282] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] created_port_ids = self._update_ports_for_instance( [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] with excutils.save_and_reraise_exception(): [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self.force_reraise() [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] raise self.value [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] updated_port = self._update_port( [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] _ensure_no_port_binding_failure(port) [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1949.004719] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] raise exception.PortBindingFailed(port_id=port['id']) [ 1949.005089] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1949.005089] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] [ 1949.005089] env[62506]: INFO nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Terminating instance [ 1949.021984] env[62506]: DEBUG nova.network.neutron [-] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1949.073700] env[62506]: DEBUG nova.network.neutron [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1949.412152] env[62506]: DEBUG nova.scheduler.client.report [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1949.508798] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquiring lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1949.509055] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquired lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1949.509602] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1949.525441] env[62506]: INFO nova.compute.manager [-] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Took 1.03 seconds to deallocate network for instance. [ 1949.528173] env[62506]: DEBUG nova.compute.claims [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1949.528524] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1949.574992] env[62506]: INFO nova.compute.manager [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] [instance: 43d09ce7-7946-43d1-8729-d7068c1cdec4] Took 1.03 seconds to deallocate network for instance. [ 1949.914746] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.479s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1949.915380] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1949.918009] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.056s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1949.919491] env[62506]: INFO nova.compute.claims [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1950.023252] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "1970c9fb-368f-4fb5-80be-1b8601124bfe" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1950.023573] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "1970c9fb-368f-4fb5-80be-1b8601124bfe" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1950.031042] env[62506]: DEBUG nova.compute.manager [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Received event network-changed-60828df4-606a-4cb9-aa71-e54ffe880915 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1950.031282] env[62506]: DEBUG nova.compute.manager [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Refreshing instance network info cache due to event network-changed-60828df4-606a-4cb9-aa71-e54ffe880915. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1950.031513] env[62506]: DEBUG oslo_concurrency.lockutils [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] Acquiring lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1950.046048] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1950.143483] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1950.425023] env[62506]: DEBUG nova.compute.utils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1950.429819] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1950.429922] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1950.495876] env[62506]: DEBUG nova.policy [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3d78efd7563a4575bbfbff61ce8d6177', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '80010f97bf374725b3346e9d9633aea0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1950.610681] env[62506]: INFO nova.scheduler.client.report [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Deleted allocations for instance 43d09ce7-7946-43d1-8729-d7068c1cdec4 [ 1950.646248] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Releasing lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1950.646826] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1950.646920] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1950.647203] env[62506]: DEBUG oslo_concurrency.lockutils [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] Acquired lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1950.647379] env[62506]: DEBUG nova.network.neutron [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Refreshing network info cache for port 60828df4-606a-4cb9-aa71-e54ffe880915 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1950.648352] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-289a7c04-806e-4aa9-905c-d6ae208679bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1950.661419] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d272e7d-be75-43d8-b5bd-dd807015cd06 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1950.695472] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 08cc6c54-48f2-4064-93a4-4debb98ab606 could not be found. [ 1950.695728] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1950.695914] env[62506]: INFO nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1950.696179] env[62506]: DEBUG oslo.service.loopingcall [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1950.696422] env[62506]: DEBUG nova.compute.manager [-] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1950.696520] env[62506]: DEBUG nova.network.neutron [-] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1950.715452] env[62506]: DEBUG nova.network.neutron [-] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1950.872815] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Successfully created port: befb19d2-7017-47b2-a528-4c598c945288 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1950.934708] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1951.122761] env[62506]: DEBUG oslo_concurrency.lockutils [None req-17eef205-82d3-4830-b656-fd5a1c40b0b0 tempest-InstanceActionsNegativeTestJSON-2083604730 tempest-InstanceActionsNegativeTestJSON-2083604730-project-member] Lock "43d09ce7-7946-43d1-8729-d7068c1cdec4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 109.295s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1951.197080] env[62506]: DEBUG nova.network.neutron [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1951.217669] env[62506]: DEBUG nova.network.neutron [-] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1951.240197] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bae1956f-1e54-4142-93e4-0ab0c9530566 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1951.247679] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14a834a2-697e-429c-8704-406f556e232f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1951.282790] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cecfd9c2-13f3-4209-b49d-0b80bb8ca043 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1951.291606] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d85143f-28fa-48bd-bc5f-5bcfaafc2321 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1951.306724] env[62506]: DEBUG nova.compute.provider_tree [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1951.347614] env[62506]: DEBUG nova.network.neutron [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1951.626689] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1951.723161] env[62506]: INFO nova.compute.manager [-] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Took 1.03 seconds to deallocate network for instance. [ 1951.728393] env[62506]: DEBUG nova.compute.claims [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1951.728836] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1951.813228] env[62506]: DEBUG nova.scheduler.client.report [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1951.851219] env[62506]: DEBUG oslo_concurrency.lockutils [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] Releasing lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1951.851219] env[62506]: DEBUG nova.compute.manager [req-62c58608-a887-439f-8cdd-35aa076f3e9e req-8d0ffc07-87e6-4cff-bf89-058e4f3ae207 service nova] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Received event network-vif-deleted-60828df4-606a-4cb9-aa71-e54ffe880915 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1951.949097] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1951.976142] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1951.976142] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1951.976142] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1951.976298] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1951.976298] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1951.976298] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1951.976298] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1951.976298] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1951.976445] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1951.976445] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1951.976852] env[62506]: DEBUG nova.virt.hardware [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1951.977835] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53683675-4482-4f22-94cf-e28acead33eb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1951.989239] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d7b32e3-c8c7-4c98-817b-fadcf7799d4e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1951.995224] env[62506]: ERROR nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1951.995224] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1951.995224] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1951.995224] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1951.995224] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1951.995224] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1951.995224] env[62506]: ERROR nova.compute.manager raise self.value [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1951.995224] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1951.995224] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1951.995224] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1951.995747] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1951.995747] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1951.995747] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1951.995747] env[62506]: ERROR nova.compute.manager [ 1951.995747] env[62506]: Traceback (most recent call last): [ 1951.995747] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1951.995747] env[62506]: listener.cb(fileno) [ 1951.995747] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1951.995747] env[62506]: result = function(*args, **kwargs) [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1951.995747] env[62506]: return func(*args, **kwargs) [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1951.995747] env[62506]: raise e [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1951.995747] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1951.995747] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1951.995747] env[62506]: with excutils.save_and_reraise_exception(): [ 1951.995747] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1951.995747] env[62506]: self.force_reraise() [ 1951.995747] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1951.995747] env[62506]: raise self.value [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1951.995747] env[62506]: updated_port = self._update_port( [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1951.995747] env[62506]: _ensure_no_port_binding_failure(port) [ 1951.995747] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1951.995747] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1951.996711] env[62506]: nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1951.996711] env[62506]: Removing descriptor: 15 [ 1952.006809] env[62506]: ERROR nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Traceback (most recent call last): [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] yield resources [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self.driver.spawn(context, instance, image_meta, [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] vm_ref = self.build_virtual_machine(instance, [ 1952.006809] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] vif_infos = vmwarevif.get_vif_info(self._session, [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] for vif in network_info: [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] return self._sync_wrapper(fn, *args, **kwargs) [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self.wait() [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self[:] = self._gt.wait() [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] return self._exit_event.wait() [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1952.007286] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] current.throw(*self._exc) [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] result = function(*args, **kwargs) [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] return func(*args, **kwargs) [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] raise e [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] nwinfo = self.network_api.allocate_for_instance( [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] created_port_ids = self._update_ports_for_instance( [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] with excutils.save_and_reraise_exception(): [ 1952.007695] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self.force_reraise() [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] raise self.value [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] updated_port = self._update_port( [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] _ensure_no_port_binding_failure(port) [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] raise exception.PortBindingFailed(port_id=port['id']) [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1952.008160] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] [ 1952.008160] env[62506]: INFO nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Terminating instance [ 1952.056545] env[62506]: DEBUG nova.compute.manager [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Received event network-changed-befb19d2-7017-47b2-a528-4c598c945288 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1952.056697] env[62506]: DEBUG nova.compute.manager [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Refreshing instance network info cache due to event network-changed-befb19d2-7017-47b2-a528-4c598c945288. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1952.056925] env[62506]: DEBUG oslo_concurrency.lockutils [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] Acquiring lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1952.057053] env[62506]: DEBUG oslo_concurrency.lockutils [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] Acquired lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1952.057183] env[62506]: DEBUG nova.network.neutron [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Refreshing network info cache for port befb19d2-7017-47b2-a528-4c598c945288 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1952.152381] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1952.317714] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.400s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1952.318267] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1952.322035] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.374s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1952.511397] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquiring lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1952.594846] env[62506]: DEBUG nova.network.neutron [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1952.826733] env[62506]: DEBUG nova.compute.utils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1952.832023] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1952.832023] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1952.878809] env[62506]: DEBUG nova.network.neutron [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1952.891020] env[62506]: DEBUG nova.policy [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c871f6225b2c45828acc0305d7bc8d2d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0be23177b93a45a5a5a632ea344a4aa0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1953.108912] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97316c9a-98fd-438c-9b43-bd2ec45f3c57 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1953.118937] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-529c89f1-432b-44cf-9424-152b97d5dd93 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1953.149930] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-734ef12e-c6b0-4023-9b64-789e13094ddd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1953.158397] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b0a31c8-0eab-4cab-94f9-49d4ec9e56f5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1953.172920] env[62506]: DEBUG nova.compute.provider_tree [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1953.205167] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Successfully created port: 433f07a7-4387-40b5-91e7-a64a211149b0 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1953.331293] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1953.381220] env[62506]: DEBUG oslo_concurrency.lockutils [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] Releasing lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1953.381501] env[62506]: DEBUG nova.compute.manager [req-e1161c2d-44d8-4d65-8a7d-4c67861dd3c5 req-f92ec096-dcb9-4b38-bf6a-03be8adf70c8 service nova] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Received event network-vif-deleted-befb19d2-7017-47b2-a528-4c598c945288 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1953.381853] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquired lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1953.382038] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1953.676239] env[62506]: DEBUG nova.scheduler.client.report [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1953.900515] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1953.917316] env[62506]: DEBUG nova.compute.manager [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Received event network-changed-433f07a7-4387-40b5-91e7-a64a211149b0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1953.917597] env[62506]: DEBUG nova.compute.manager [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Refreshing instance network info cache due to event network-changed-433f07a7-4387-40b5-91e7-a64a211149b0. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1953.917854] env[62506]: DEBUG oslo_concurrency.lockutils [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] Acquiring lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1953.918094] env[62506]: DEBUG oslo_concurrency.lockutils [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] Acquired lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1953.918300] env[62506]: DEBUG nova.network.neutron [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Refreshing network info cache for port 433f07a7-4387-40b5-91e7-a64a211149b0 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1954.025421] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1954.091603] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "5e16bc57-424c-45b0-8e85-1d80bac92b18" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1954.091864] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "5e16bc57-424c-45b0-8e85-1d80bac92b18" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1954.093434] env[62506]: ERROR nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1954.093434] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1954.093434] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1954.093434] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1954.093434] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1954.093434] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1954.093434] env[62506]: ERROR nova.compute.manager raise self.value [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1954.093434] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1954.093434] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1954.093434] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1954.093915] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1954.093915] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1954.093915] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1954.093915] env[62506]: ERROR nova.compute.manager [ 1954.093915] env[62506]: Traceback (most recent call last): [ 1954.093915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1954.093915] env[62506]: listener.cb(fileno) [ 1954.093915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1954.093915] env[62506]: result = function(*args, **kwargs) [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1954.093915] env[62506]: return func(*args, **kwargs) [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1954.093915] env[62506]: raise e [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1954.093915] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1954.093915] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1954.093915] env[62506]: with excutils.save_and_reraise_exception(): [ 1954.093915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1954.093915] env[62506]: self.force_reraise() [ 1954.093915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1954.093915] env[62506]: raise self.value [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1954.093915] env[62506]: updated_port = self._update_port( [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1954.093915] env[62506]: _ensure_no_port_binding_failure(port) [ 1954.093915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1954.093915] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1954.094986] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1954.094986] env[62506]: Removing descriptor: 20 [ 1954.182215] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.860s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1954.182880] env[62506]: ERROR nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Traceback (most recent call last): [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self.driver.spawn(context, instance, image_meta, [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] vm_ref = self.build_virtual_machine(instance, [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] vif_infos = vmwarevif.get_vif_info(self._session, [ 1954.182880] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] for vif in network_info: [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return self._sync_wrapper(fn, *args, **kwargs) [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self.wait() [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self[:] = self._gt.wait() [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return self._exit_event.wait() [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] result = hub.switch() [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1954.183319] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return self.greenlet.switch() [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] result = function(*args, **kwargs) [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] return func(*args, **kwargs) [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] raise e [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] nwinfo = self.network_api.allocate_for_instance( [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] created_port_ids = self._update_ports_for_instance( [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] with excutils.save_and_reraise_exception(): [ 1954.183720] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] self.force_reraise() [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] raise self.value [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] updated_port = self._update_port( [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] _ensure_no_port_binding_failure(port) [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] raise exception.PortBindingFailed(port_id=port['id']) [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] nova.exception.PortBindingFailed: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. [ 1954.184131] env[62506]: ERROR nova.compute.manager [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] [ 1954.184477] env[62506]: DEBUG nova.compute.utils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1954.184779] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.241s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1954.187745] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Build of instance 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9 was re-scheduled: Binding failed for port be9686dd-6786-49af-9994-03f259ddac8b, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1954.188194] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1954.188418] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquiring lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1954.188564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Acquired lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1954.188744] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1954.286284] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "8665bdf3-3930-4f45-b5be-e308f8f53828" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1954.286510] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "8665bdf3-3930-4f45-b5be-e308f8f53828" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1954.340658] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1954.364576] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1954.364817] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1954.364998] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1954.365210] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1954.365357] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1954.365503] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1954.365704] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1954.365863] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1954.366039] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1954.366207] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1954.366379] env[62506]: DEBUG nova.virt.hardware [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1954.367232] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-441a14fa-c3fd-4a43-92fe-ed18ae0ee4d6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.376194] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8d93b45-a0b7-4d23-a657-d1866a6cb341 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.390053] env[62506]: ERROR nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Traceback (most recent call last): [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] yield resources [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self.driver.spawn(context, instance, image_meta, [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] vm_ref = self.build_virtual_machine(instance, [ 1954.390053] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] vif_infos = vmwarevif.get_vif_info(self._session, [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] for vif in network_info: [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] return self._sync_wrapper(fn, *args, **kwargs) [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self.wait() [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self[:] = self._gt.wait() [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] return self._exit_event.wait() [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1954.390446] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] current.throw(*self._exc) [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] result = function(*args, **kwargs) [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] return func(*args, **kwargs) [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] raise e [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] nwinfo = self.network_api.allocate_for_instance( [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] created_port_ids = self._update_ports_for_instance( [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] with excutils.save_and_reraise_exception(): [ 1954.390882] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self.force_reraise() [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] raise self.value [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] updated_port = self._update_port( [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] _ensure_no_port_binding_failure(port) [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] raise exception.PortBindingFailed(port_id=port['id']) [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1954.391326] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] [ 1954.391326] env[62506]: INFO nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Terminating instance [ 1954.435802] env[62506]: DEBUG nova.network.neutron [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1954.509219] env[62506]: DEBUG nova.network.neutron [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1954.528218] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Releasing lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1954.528639] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1954.528856] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1954.529153] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9dd8db4f-c61f-4256-9a82-ccab32d51238 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.539204] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f0f41b1-99bb-435c-b294-aa3b16db17c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.563376] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3d034f40-5896-49bc-bed1-9062e1fde1e3 could not be found. [ 1954.563596] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1954.563873] env[62506]: INFO nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1954.564223] env[62506]: DEBUG oslo.service.loopingcall [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1954.564543] env[62506]: DEBUG nova.compute.manager [-] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1954.564704] env[62506]: DEBUG nova.network.neutron [-] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1954.579451] env[62506]: DEBUG nova.network.neutron [-] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1954.712684] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1954.796199] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1954.893847] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquiring lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1954.949672] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-206d5f2f-5e42-4d11-9d50-229479a343d2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.957909] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ce8cfb5-902d-430c-9b6d-df87f2fda639 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.987777] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d02f3e14-9e72-4ce3-add9-2032b86f5b80 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.995575] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45f45a40-ac4c-4964-96a2-58eea522a5b0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.008591] env[62506]: DEBUG nova.compute.provider_tree [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1955.012104] env[62506]: DEBUG oslo_concurrency.lockutils [req-97597cc7-fdc0-4ff5-964f-c58efc813b35 req-92cece80-10a7-447e-b6b9-79dc6212ab13 service nova] Releasing lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1955.012474] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquired lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1955.012656] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1955.082148] env[62506]: DEBUG nova.network.neutron [-] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1955.298881] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Releasing lock "refresh_cache-7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1955.299120] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1955.299446] env[62506]: DEBUG nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1955.299624] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1955.315355] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1955.512283] env[62506]: DEBUG nova.scheduler.client.report [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1955.530601] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1955.585112] env[62506]: INFO nova.compute.manager [-] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Took 1.02 seconds to deallocate network for instance. [ 1955.587634] env[62506]: DEBUG nova.compute.claims [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1955.587805] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1955.602653] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1955.819226] env[62506]: DEBUG nova.network.neutron [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1955.944446] env[62506]: DEBUG nova.compute.manager [req-37425609-c16f-4a7d-84d3-0da9c0e3bf2b req-88db5327-19cf-4819-8092-988c8a6416e0 service nova] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Received event network-vif-deleted-433f07a7-4387-40b5-91e7-a64a211149b0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1956.020075] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.834s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1956.020075] env[62506]: ERROR nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Traceback (most recent call last): [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self.driver.spawn(context, instance, image_meta, [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1956.020075] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] vm_ref = self.build_virtual_machine(instance, [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] vif_infos = vmwarevif.get_vif_info(self._session, [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] for vif in network_info: [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return self._sync_wrapper(fn, *args, **kwargs) [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self.wait() [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self[:] = self._gt.wait() [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return self._exit_event.wait() [ 1956.020649] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] result = hub.switch() [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return self.greenlet.switch() [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] result = function(*args, **kwargs) [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] return func(*args, **kwargs) [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] raise e [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] nwinfo = self.network_api.allocate_for_instance( [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1956.021104] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] created_port_ids = self._update_ports_for_instance( [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] with excutils.save_and_reraise_exception(): [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] self.force_reraise() [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] raise self.value [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] updated_port = self._update_port( [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] _ensure_no_port_binding_failure(port) [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1956.021523] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] raise exception.PortBindingFailed(port_id=port['id']) [ 1956.021973] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] nova.exception.PortBindingFailed: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. [ 1956.021973] env[62506]: ERROR nova.compute.manager [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] [ 1956.021973] env[62506]: DEBUG nova.compute.utils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1956.022519] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.267s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1956.025295] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Build of instance 01ad7fbe-7ea0-425b-ba63-28537cb158b1 was re-scheduled: Binding failed for port 485593e6-f77a-42f4-ae45-94d9e51d1679, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1956.025744] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1956.025968] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1956.026144] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquired lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1956.026302] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1956.106040] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Releasing lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1956.106241] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1956.106437] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1956.106743] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-268bb717-c632-4224-806c-737cfa0cfdf2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.116327] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f3c6db1-124b-4980-91dd-5a453fd89580 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.139567] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b5d6b647-e24f-4e53-b448-0614334fd760 could not be found. [ 1956.139773] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1956.139956] env[62506]: INFO nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1956.140214] env[62506]: DEBUG oslo.service.loopingcall [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1956.140448] env[62506]: DEBUG nova.compute.manager [-] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1956.140545] env[62506]: DEBUG nova.network.neutron [-] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1956.157862] env[62506]: DEBUG nova.network.neutron [-] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1956.321731] env[62506]: INFO nova.compute.manager [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] [instance: 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9] Took 1.02 seconds to deallocate network for instance. [ 1956.550733] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1956.609882] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1956.661530] env[62506]: DEBUG nova.network.neutron [-] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1956.840999] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7f9aad7-327d-4cc7-a5ad-9a8660f526db {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.849136] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b41d3e76-1007-4019-8483-16e5549c7348 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.883431] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12a882a9-f334-4a35-a466-ff2cd72eccda {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.891943] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-099ed98c-0d12-43f0-9489-12d0d7642164 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.906152] env[62506]: DEBUG nova.compute.provider_tree [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1957.113233] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Releasing lock "refresh_cache-01ad7fbe-7ea0-425b-ba63-28537cb158b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1957.113531] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1957.113639] env[62506]: DEBUG nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1957.113804] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1957.128396] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1957.164471] env[62506]: INFO nova.compute.manager [-] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Took 1.02 seconds to deallocate network for instance. [ 1957.166761] env[62506]: DEBUG nova.compute.claims [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1957.166942] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1957.354340] env[62506]: INFO nova.scheduler.client.report [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Deleted allocations for instance 7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9 [ 1957.408984] env[62506]: DEBUG nova.scheduler.client.report [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1957.631584] env[62506]: DEBUG nova.network.neutron [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1957.862605] env[62506]: DEBUG oslo_concurrency.lockutils [None req-82b1dfa4-e463-459c-b643-ab858136e4a0 tempest-InstanceActionsTestJSON-1496833818 tempest-InstanceActionsTestJSON-1496833818-project-member] Lock "7a9be56e-f93e-46d5-ac7b-09ab2a44e7f9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 108.675s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1957.914283] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.891s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1957.914283] env[62506]: ERROR nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Traceback (most recent call last): [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self.driver.spawn(context, instance, image_meta, [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1957.914283] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] vm_ref = self.build_virtual_machine(instance, [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] vif_infos = vmwarevif.get_vif_info(self._session, [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] for vif in network_info: [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return self._sync_wrapper(fn, *args, **kwargs) [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self.wait() [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self[:] = self._gt.wait() [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return self._exit_event.wait() [ 1957.914624] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] result = hub.switch() [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return self.greenlet.switch() [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] result = function(*args, **kwargs) [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] return func(*args, **kwargs) [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] raise e [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] nwinfo = self.network_api.allocate_for_instance( [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1957.915037] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] created_port_ids = self._update_ports_for_instance( [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] with excutils.save_and_reraise_exception(): [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] self.force_reraise() [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] raise self.value [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] updated_port = self._update_port( [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] _ensure_no_port_binding_failure(port) [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1957.915511] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] raise exception.PortBindingFailed(port_id=port['id']) [ 1957.915965] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] nova.exception.PortBindingFailed: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. [ 1957.915965] env[62506]: ERROR nova.compute.manager [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] [ 1957.915965] env[62506]: DEBUG nova.compute.utils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1957.916104] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.362s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1957.917523] env[62506]: INFO nova.compute.claims [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1957.920381] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Build of instance 854c955f-b327-4c25-87bd-3ad68dbff024 was re-scheduled: Binding failed for port bdaecc2f-5bee-4b03-b42c-ea028e363d10, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1957.921055] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1957.921055] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquiring lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1957.921196] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Acquired lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1957.921271] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1958.133727] env[62506]: INFO nova.compute.manager [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 01ad7fbe-7ea0-425b-ba63-28537cb158b1] Took 1.02 seconds to deallocate network for instance. [ 1958.365524] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1958.442571] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1958.673623] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1958.894833] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1959.173929] env[62506]: INFO nova.scheduler.client.report [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Deleted allocations for instance 01ad7fbe-7ea0-425b-ba63-28537cb158b1 [ 1959.181559] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Releasing lock "refresh_cache-854c955f-b327-4c25-87bd-3ad68dbff024" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1959.181738] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1959.186027] env[62506]: DEBUG nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1959.186027] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1959.213584] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1959.259911] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69a49ab4-c655-4815-9fac-f9098428b01f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.269719] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93f9abf5-9151-45fa-9899-b1c6c0cc4155 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.303464] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bee25a37-ffac-4c1d-9e84-963a3a9a97e6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.313523] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-242548ff-d584-4966-a71b-1b6c43f718d5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1959.328996] env[62506]: DEBUG nova.compute.provider_tree [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1959.689523] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f190a5b1-839e-478b-97e6-def1ec264866 tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "01ad7fbe-7ea0-425b-ba63-28537cb158b1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 107.721s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1959.716188] env[62506]: DEBUG nova.network.neutron [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1959.832846] env[62506]: DEBUG nova.scheduler.client.report [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1959.881238] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquiring lock "dc0c0209-786b-44db-8f06-a429eec28bde" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1959.881581] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Lock "dc0c0209-786b-44db-8f06-a429eec28bde" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1960.192985] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1960.219432] env[62506]: INFO nova.compute.manager [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] [instance: 854c955f-b327-4c25-87bd-3ad68dbff024] Took 1.04 seconds to deallocate network for instance. [ 1960.339916] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.424s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1960.340472] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1960.343072] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.784s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1960.344603] env[62506]: INFO nova.compute.claims [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1960.715818] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1960.854964] env[62506]: DEBUG nova.compute.utils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1960.856414] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1960.856584] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1960.904279] env[62506]: DEBUG nova.policy [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'efde00890e9d48409d02d4b9b7bcf67c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cc89c3a96fd8408f9545ec3f3023a75d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1961.185635] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Successfully created port: d0b4e330-33dc-466c-8dcc-d5a60fefc322 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1961.252607] env[62506]: INFO nova.scheduler.client.report [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Deleted allocations for instance 854c955f-b327-4c25-87bd-3ad68dbff024 [ 1961.364555] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1961.637292] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cebe3c76-dac5-4bc7-bfe4-668a988eb9de {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1961.645895] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09dac900-7bf4-4dcb-854f-768e56f66337 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1961.678101] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1fcc1e1-39ae-41d7-959a-1515d8136baa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1961.685788] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-220d6bb3-3ad7-42d1-a4d1-07dfce89f3fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1961.699242] env[62506]: DEBUG nova.compute.provider_tree [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1961.761177] env[62506]: DEBUG oslo_concurrency.lockutils [None req-15bb094a-170c-4aa1-b49c-e27325016e7f tempest-ListImageFiltersTestJSON-1099151808 tempest-ListImageFiltersTestJSON-1099151808-project-member] Lock "854c955f-b327-4c25-87bd-3ad68dbff024" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 108.929s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1961.920380] env[62506]: DEBUG nova.compute.manager [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Received event network-changed-d0b4e330-33dc-466c-8dcc-d5a60fefc322 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1961.920654] env[62506]: DEBUG nova.compute.manager [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Refreshing instance network info cache due to event network-changed-d0b4e330-33dc-466c-8dcc-d5a60fefc322. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1961.920795] env[62506]: DEBUG oslo_concurrency.lockutils [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] Acquiring lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1961.921373] env[62506]: DEBUG oslo_concurrency.lockutils [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] Acquired lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1961.921373] env[62506]: DEBUG nova.network.neutron [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Refreshing network info cache for port d0b4e330-33dc-466c-8dcc-d5a60fefc322 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1962.043337] env[62506]: ERROR nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1962.043337] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1962.043337] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1962.043337] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1962.043337] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1962.043337] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1962.043337] env[62506]: ERROR nova.compute.manager raise self.value [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1962.043337] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1962.043337] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1962.043337] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1962.043915] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1962.043915] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1962.043915] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1962.043915] env[62506]: ERROR nova.compute.manager [ 1962.043915] env[62506]: Traceback (most recent call last): [ 1962.043915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1962.043915] env[62506]: listener.cb(fileno) [ 1962.043915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1962.043915] env[62506]: result = function(*args, **kwargs) [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1962.043915] env[62506]: return func(*args, **kwargs) [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1962.043915] env[62506]: raise e [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1962.043915] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1962.043915] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1962.043915] env[62506]: with excutils.save_and_reraise_exception(): [ 1962.043915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1962.043915] env[62506]: self.force_reraise() [ 1962.043915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1962.043915] env[62506]: raise self.value [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1962.043915] env[62506]: updated_port = self._update_port( [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1962.043915] env[62506]: _ensure_no_port_binding_failure(port) [ 1962.043915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1962.043915] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1962.044877] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1962.044877] env[62506]: Removing descriptor: 20 [ 1962.202576] env[62506]: DEBUG nova.scheduler.client.report [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1962.266896] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1962.374164] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1962.396768] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1962.396768] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1962.396950] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1962.397101] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1962.397252] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1962.397399] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1962.397608] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1962.397767] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1962.397932] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1962.398114] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1962.398290] env[62506]: DEBUG nova.virt.hardware [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1962.399196] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-726855a9-e985-4b32-997e-6b5b21707a22 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1962.407076] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9898346a-65b0-401e-8031-9e7511661c8a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1962.422279] env[62506]: ERROR nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Traceback (most recent call last): [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] yield resources [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self.driver.spawn(context, instance, image_meta, [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] vm_ref = self.build_virtual_machine(instance, [ 1962.422279] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] vif_infos = vmwarevif.get_vif_info(self._session, [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] for vif in network_info: [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] return self._sync_wrapper(fn, *args, **kwargs) [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self.wait() [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self[:] = self._gt.wait() [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] return self._exit_event.wait() [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1962.422695] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] current.throw(*self._exc) [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] result = function(*args, **kwargs) [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] return func(*args, **kwargs) [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] raise e [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] nwinfo = self.network_api.allocate_for_instance( [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] created_port_ids = self._update_ports_for_instance( [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] with excutils.save_and_reraise_exception(): [ 1962.423081] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self.force_reraise() [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] raise self.value [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] updated_port = self._update_port( [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] _ensure_no_port_binding_failure(port) [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] raise exception.PortBindingFailed(port_id=port['id']) [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1962.423605] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] [ 1962.423605] env[62506]: INFO nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Terminating instance [ 1962.440612] env[62506]: DEBUG nova.network.neutron [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1962.522874] env[62506]: DEBUG nova.network.neutron [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1962.708080] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.365s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1962.708651] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1962.711564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 17.111s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1962.794334] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1962.932313] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquiring lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1963.025250] env[62506]: DEBUG oslo_concurrency.lockutils [req-0b84bdda-cd1c-4c85-906f-8ae76f46d3c7 req-7cf6f9fb-e3d8-411c-a552-5683c9a69305 service nova] Releasing lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1963.025670] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquired lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1963.025861] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1963.215861] env[62506]: DEBUG nova.compute.utils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1963.217282] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1963.217452] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1963.261660] env[62506]: DEBUG nova.policy [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f73527aacb5c43e7b78c5bdcfe24ed68', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'eb1f9277c5494d3f90e5c70248189824', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1963.546315] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1963.552054] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Successfully created port: fd65fd9a-cb71-42a8-b33f-91460241309d {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1963.724288] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1963.764198] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 6cdba5b2-af96-4766-9a97-3a758983c49b actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1963.764198] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 08cc6c54-48f2-4064-93a4-4debb98ab606 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1963.764198] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 3d034f40-5896-49bc-bed1-9062e1fde1e3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1963.764198] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b5d6b647-e24f-4e53-b448-0614334fd760 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1963.764577] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dcd5a800-1a28-45b3-8eff-a2e8cb871491 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1963.764577] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 04218b42-91dd-4045-8a40-f8829910e064 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1963.768525] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1963.947324] env[62506]: DEBUG nova.compute.manager [req-a9c40f71-df7b-483e-8a67-589270a0b0a3 req-0e63ac67-684b-472d-b4f3-9ff6de267817 service nova] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Received event network-vif-deleted-d0b4e330-33dc-466c-8dcc-d5a60fefc322 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1964.270113] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance ecc8aab4-a172-4218-8b09-ccfde0d03903 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1964.271723] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Releasing lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1964.273354] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1964.273596] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1964.275365] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1dba56f9-cb46-4740-bf54-4b34b7ebfab1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1964.286947] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a01e92a5-7f09-4f36-b7d4-404ab96bf37b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1964.311254] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dcd5a800-1a28-45b3-8eff-a2e8cb871491 could not be found. [ 1964.311646] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1964.311974] env[62506]: INFO nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1964.312414] env[62506]: DEBUG oslo.service.loopingcall [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1964.312730] env[62506]: DEBUG nova.compute.manager [-] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1964.312945] env[62506]: DEBUG nova.network.neutron [-] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1964.340435] env[62506]: DEBUG nova.network.neutron [-] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1964.387574] env[62506]: ERROR nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1964.387574] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1964.387574] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1964.387574] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1964.387574] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1964.387574] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1964.387574] env[62506]: ERROR nova.compute.manager raise self.value [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1964.387574] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1964.387574] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1964.387574] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1964.388138] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1964.388138] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1964.388138] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1964.388138] env[62506]: ERROR nova.compute.manager [ 1964.388138] env[62506]: Traceback (most recent call last): [ 1964.388138] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1964.388138] env[62506]: listener.cb(fileno) [ 1964.388138] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1964.388138] env[62506]: result = function(*args, **kwargs) [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1964.388138] env[62506]: return func(*args, **kwargs) [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1964.388138] env[62506]: raise e [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1964.388138] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1964.388138] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1964.388138] env[62506]: with excutils.save_and_reraise_exception(): [ 1964.388138] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1964.388138] env[62506]: self.force_reraise() [ 1964.388138] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1964.388138] env[62506]: raise self.value [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1964.388138] env[62506]: updated_port = self._update_port( [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1964.388138] env[62506]: _ensure_no_port_binding_failure(port) [ 1964.388138] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1964.388138] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1964.389086] env[62506]: nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1964.389086] env[62506]: Removing descriptor: 20 [ 1964.739419] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1964.772139] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1964.772388] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1964.774019] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1964.774019] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1964.774019] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1964.774019] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1964.774019] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1964.774349] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1964.774349] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1964.774349] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1964.774349] env[62506]: DEBUG nova.virt.hardware [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1964.774671] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd1ebc6d-ec46-43c7-8b52-fd76d5be9134 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1964.777823] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 19afb920-58b0-4ad9-be99-d0a8e930c680 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1964.785068] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14936486-9ea9-4763-b764-3ad69dcf24f4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1964.799030] env[62506]: ERROR nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] Traceback (most recent call last): [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] yield resources [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self.driver.spawn(context, instance, image_meta, [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] vm_ref = self.build_virtual_machine(instance, [ 1964.799030] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] vif_infos = vmwarevif.get_vif_info(self._session, [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] for vif in network_info: [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] return self._sync_wrapper(fn, *args, **kwargs) [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self.wait() [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self[:] = self._gt.wait() [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] return self._exit_event.wait() [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1964.799610] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] current.throw(*self._exc) [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] result = function(*args, **kwargs) [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] return func(*args, **kwargs) [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] raise e [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] nwinfo = self.network_api.allocate_for_instance( [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] created_port_ids = self._update_ports_for_instance( [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] with excutils.save_and_reraise_exception(): [ 1964.800551] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self.force_reraise() [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] raise self.value [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] updated_port = self._update_port( [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] _ensure_no_port_binding_failure(port) [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] raise exception.PortBindingFailed(port_id=port['id']) [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1964.801272] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] [ 1964.801272] env[62506]: INFO nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Terminating instance [ 1964.845987] env[62506]: DEBUG nova.network.neutron [-] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1965.285164] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dc1c594f-7625-495a-8948-cee046bf204a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1965.303263] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1965.303476] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquired lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1965.303650] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1965.351068] env[62506]: INFO nova.compute.manager [-] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Took 1.04 seconds to deallocate network for instance. [ 1965.353682] env[62506]: DEBUG nova.compute.claims [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1965.353682] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1965.788625] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e3c3759c-81e6-415d-a5a5-b17cc16bdc2f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1965.827829] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1965.922362] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1966.021425] env[62506]: DEBUG nova.compute.manager [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Received event network-changed-fd65fd9a-cb71-42a8-b33f-91460241309d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1966.021618] env[62506]: DEBUG nova.compute.manager [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Refreshing instance network info cache due to event network-changed-fd65fd9a-cb71-42a8-b33f-91460241309d. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1966.021814] env[62506]: DEBUG oslo_concurrency.lockutils [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] Acquiring lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1966.292143] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8f808943-6107-49d9-966f-c4e6ae0cdac4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1966.425022] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Releasing lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1966.425022] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1966.425022] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1966.425277] env[62506]: DEBUG oslo_concurrency.lockutils [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] Acquired lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1966.425277] env[62506]: DEBUG nova.network.neutron [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Refreshing network info cache for port fd65fd9a-cb71-42a8-b33f-91460241309d {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1966.426287] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ac2c433d-59cc-4333-9e29-c6caac825433 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1966.436451] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-619c4d96-324f-44e4-b7af-5d3f25ced8e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1966.460742] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 04218b42-91dd-4045-8a40-f8829910e064 could not be found. [ 1966.461080] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1966.461254] env[62506]: INFO nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1966.461487] env[62506]: DEBUG oslo.service.loopingcall [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1966.461695] env[62506]: DEBUG nova.compute.manager [-] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1966.461790] env[62506]: DEBUG nova.network.neutron [-] [instance: 04218b42-91dd-4045-8a40-f8829910e064] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1966.478478] env[62506]: DEBUG nova.network.neutron [-] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1966.503608] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "97091b4f-8448-49e5-9382-99e3358300c6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1966.503839] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "97091b4f-8448-49e5-9382-99e3358300c6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1966.527030] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "50b8f972-e2bd-4526-ab28-bf887a240421" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1966.527206] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "50b8f972-e2bd-4526-ab28-bf887a240421" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1966.796123] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1966.947639] env[62506]: DEBUG nova.network.neutron [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1966.981079] env[62506]: DEBUG nova.network.neutron [-] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1967.032637] env[62506]: DEBUG nova.network.neutron [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1967.298568] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance c1943459-83bf-4461-80e0-d98da37b5d17 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1967.483547] env[62506]: INFO nova.compute.manager [-] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Took 1.02 seconds to deallocate network for instance. [ 1967.485814] env[62506]: DEBUG nova.compute.claims [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1967.485988] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1967.535688] env[62506]: DEBUG oslo_concurrency.lockutils [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] Releasing lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1967.535934] env[62506]: DEBUG nova.compute.manager [req-4180b612-1ae4-4a8e-b32c-828fa0e03e14 req-597c0851-8272-4e21-8e68-6a99151ad227 service nova] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Received event network-vif-deleted-fd65fd9a-cb71-42a8-b33f-91460241309d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1967.801735] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1968.305099] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 85f24cb7-a0c3-4598-9992-2726320ca76d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1968.807605] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 3e30fb9a-092b-4886-b496-9a54cd4477d2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1969.312759] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 1970c9fb-368f-4fb5-80be-1b8601124bfe has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1969.817749] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 5e16bc57-424c-45b0-8e85-1d80bac92b18 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1970.321292] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8665bdf3-3930-4f45-b5be-e308f8f53828 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1970.825054] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dc0c0209-786b-44db-8f06-a429eec28bde has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1970.825054] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1970.825388] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1971.061663] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6dff54a0-bd63-43ad-8fdd-290b57543297 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.070224] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59edbcb1-ccbd-439d-9a96-b2314fca6169 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.099624] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77278283-64af-4e3d-b9fd-ebf6fafab4fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.106903] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b15df2c3-2e8b-4c0a-a289-635b3c83f560 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1971.119432] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1971.622125] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1972.127510] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1972.127868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 9.416s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1972.128047] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.600s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1972.862344] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e579c472-e467-4e32-9891-5d3a280a6ecd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.869706] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38fbc931-7f4c-475d-9f10-66315bb22544 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.899479] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d05ef7b8-f6b1-430d-8d26-e87dc03c27e5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.906389] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5c0e08f-00a5-4c78-b8ca-aebacbcb36a6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.919236] env[62506]: DEBUG nova.compute.provider_tree [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1973.126409] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1973.126645] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1973.423296] env[62506]: DEBUG nova.scheduler.client.report [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1973.632606] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1973.632784] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 1973.632906] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 1973.928050] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.800s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1973.928734] env[62506]: ERROR nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Traceback (most recent call last): [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self.driver.spawn(context, instance, image_meta, [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] vm_ref = self.build_virtual_machine(instance, [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] vif_infos = vmwarevif.get_vif_info(self._session, [ 1973.928734] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] for vif in network_info: [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return self._sync_wrapper(fn, *args, **kwargs) [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self.wait() [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self[:] = self._gt.wait() [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return self._exit_event.wait() [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] result = hub.switch() [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1973.929312] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return self.greenlet.switch() [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] result = function(*args, **kwargs) [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] return func(*args, **kwargs) [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] raise e [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] nwinfo = self.network_api.allocate_for_instance( [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] created_port_ids = self._update_ports_for_instance( [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] with excutils.save_and_reraise_exception(): [ 1973.929682] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] self.force_reraise() [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] raise self.value [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] updated_port = self._update_port( [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] _ensure_no_port_binding_failure(port) [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] raise exception.PortBindingFailed(port_id=port['id']) [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] nova.exception.PortBindingFailed: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. [ 1973.930034] env[62506]: ERROR nova.compute.manager [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] [ 1973.930337] env[62506]: DEBUG nova.compute.utils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1973.930739] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.202s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1973.933642] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Build of instance 6cdba5b2-af96-4766-9a97-3a758983c49b was re-scheduled: Binding failed for port d52c1b94-5eb1-441a-b6e6-0a2f53e2f782, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1973.934114] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1973.934359] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1973.934501] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquired lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1973.934678] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1974.136169] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1974.136356] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1974.136489] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1974.136616] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1974.136738] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 1974.136858] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 1974.137064] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1974.137426] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1974.137602] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1974.455458] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1974.533263] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1974.667862] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ab8a0d9-afa7-4393-a988-041a2096276c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.675836] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e77a7c36-7679-4017-a15a-4afa073db567 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.706766] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f36fa458-3159-4ca2-ac2c-a610d9dc0911 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.715183] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d940e2a0-52d4-46b8-a92c-32fa6a9fc581 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.733382] env[62506]: DEBUG nova.compute.provider_tree [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1975.036163] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Releasing lock "refresh_cache-6cdba5b2-af96-4766-9a97-3a758983c49b" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1975.036327] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1975.036513] env[62506]: DEBUG nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1975.036678] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1975.051970] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1975.237327] env[62506]: DEBUG nova.scheduler.client.report [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1975.555219] env[62506]: DEBUG nova.network.neutron [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1975.742319] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.811s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1975.742964] env[62506]: ERROR nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Traceback (most recent call last): [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self.driver.spawn(context, instance, image_meta, [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] vm_ref = self.build_virtual_machine(instance, [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] vif_infos = vmwarevif.get_vif_info(self._session, [ 1975.742964] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] for vif in network_info: [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return self._sync_wrapper(fn, *args, **kwargs) [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self.wait() [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self[:] = self._gt.wait() [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return self._exit_event.wait() [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] result = hub.switch() [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1975.743489] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return self.greenlet.switch() [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] result = function(*args, **kwargs) [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] return func(*args, **kwargs) [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] raise e [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] nwinfo = self.network_api.allocate_for_instance( [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] created_port_ids = self._update_ports_for_instance( [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] with excutils.save_and_reraise_exception(): [ 1975.744043] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] self.force_reraise() [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] raise self.value [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] updated_port = self._update_port( [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] _ensure_no_port_binding_failure(port) [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] raise exception.PortBindingFailed(port_id=port['id']) [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] nova.exception.PortBindingFailed: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. [ 1975.744653] env[62506]: ERROR nova.compute.manager [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] [ 1975.745143] env[62506]: DEBUG nova.compute.utils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1975.745143] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.593s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1975.746299] env[62506]: INFO nova.compute.claims [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1975.748775] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Build of instance 08cc6c54-48f2-4064-93a4-4debb98ab606 was re-scheduled: Binding failed for port 60828df4-606a-4cb9-aa71-e54ffe880915, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1975.749284] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1975.749501] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquiring lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1975.749647] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Acquired lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1975.749803] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1976.058155] env[62506]: INFO nova.compute.manager [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 6cdba5b2-af96-4766-9a97-3a758983c49b] Took 1.02 seconds to deallocate network for instance. [ 1976.268657] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1976.339980] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1976.842794] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Releasing lock "refresh_cache-08cc6c54-48f2-4064-93a4-4debb98ab606" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1976.843079] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1976.843155] env[62506]: DEBUG nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1976.843272] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1976.860625] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1977.022342] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf2048ce-56e6-4190-b78d-282ed48423a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1977.030106] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3d38c7e-e21e-4144-a2d4-8957e931f70a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1977.059016] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-639f1c69-1cfd-407a-aa97-5de2e8c4c899 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1977.066375] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-541d373d-f945-49e8-95e5-4825dad69b38 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1977.082156] env[62506]: DEBUG nova.compute.provider_tree [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1977.088184] env[62506]: INFO nova.scheduler.client.report [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Deleted allocations for instance 6cdba5b2-af96-4766-9a97-3a758983c49b [ 1977.363995] env[62506]: DEBUG nova.network.neutron [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1977.584773] env[62506]: DEBUG nova.scheduler.client.report [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1977.595754] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8a11dfd-a452-4608-87a5-4b7bce2f5273 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "6cdba5b2-af96-4766-9a97-3a758983c49b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 118.777s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1977.865948] env[62506]: INFO nova.compute.manager [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] [instance: 08cc6c54-48f2-4064-93a4-4debb98ab606] Took 1.02 seconds to deallocate network for instance. [ 1978.089128] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.344s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1978.089720] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1978.093428] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.505s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1978.099646] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1978.594821] env[62506]: DEBUG nova.compute.utils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1978.596927] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1978.597126] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1978.617043] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1978.638279] env[62506]: DEBUG nova.policy [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3105ec2e01bb4d7a9d4bd17e759c16fd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0d5117e035074ba6aabe5e1221496593', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1978.871339] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9db4470d-fa5e-457a-bd00-a427ca35a046 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1978.882365] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f863c37c-4678-4936-8b16-9e455fb68b00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1978.912347] env[62506]: INFO nova.scheduler.client.report [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Deleted allocations for instance 08cc6c54-48f2-4064-93a4-4debb98ab606 [ 1978.920216] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d864be0a-ddfc-4f01-a96d-0494c473b593 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1978.928230] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8f68a55-f0c8-4e2e-9a8f-7a9b5235d76b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1978.944222] env[62506]: DEBUG nova.compute.provider_tree [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1978.949849] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "73ff1d78-5841-4532-a3f0-51e3f8e3314e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1978.952347] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "73ff1d78-5841-4532-a3f0-51e3f8e3314e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1979.097775] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1979.110087] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Successfully created port: e66a5f7b-3d88-48a9-8467-e93315cc6f16 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1979.423341] env[62506]: DEBUG oslo_concurrency.lockutils [None req-933cd9c3-8f19-45e9-aa34-818b6fe41505 tempest-ImagesOneServerTestJSON-1576681740 tempest-ImagesOneServerTestJSON-1576681740-project-member] Lock "08cc6c54-48f2-4064-93a4-4debb98ab606" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 119.672s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1979.450165] env[62506]: DEBUG nova.scheduler.client.report [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1979.925687] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1979.954901] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.862s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1979.955629] env[62506]: ERROR nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Traceback (most recent call last): [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self.driver.spawn(context, instance, image_meta, [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] vm_ref = self.build_virtual_machine(instance, [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] vif_infos = vmwarevif.get_vif_info(self._session, [ 1979.955629] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] for vif in network_info: [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] return self._sync_wrapper(fn, *args, **kwargs) [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self.wait() [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self[:] = self._gt.wait() [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] return self._exit_event.wait() [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] current.throw(*self._exc) [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1979.956689] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] result = function(*args, **kwargs) [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] return func(*args, **kwargs) [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] raise e [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] nwinfo = self.network_api.allocate_for_instance( [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] created_port_ids = self._update_ports_for_instance( [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] with excutils.save_and_reraise_exception(): [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] self.force_reraise() [ 1979.957733] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] raise self.value [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] updated_port = self._update_port( [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] _ensure_no_port_binding_failure(port) [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] raise exception.PortBindingFailed(port_id=port['id']) [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] nova.exception.PortBindingFailed: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. [ 1979.958099] env[62506]: ERROR nova.compute.manager [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] [ 1979.958099] env[62506]: DEBUG nova.compute.utils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1979.958385] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Build of instance 3d034f40-5896-49bc-bed1-9062e1fde1e3 was re-scheduled: Binding failed for port befb19d2-7017-47b2-a528-4c598c945288, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1979.958867] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1979.959283] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquiring lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1979.959508] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Acquired lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1979.959711] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1979.961298] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.794s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1980.087966] env[62506]: DEBUG nova.compute.manager [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Received event network-changed-e66a5f7b-3d88-48a9-8467-e93315cc6f16 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1980.088185] env[62506]: DEBUG nova.compute.manager [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Refreshing instance network info cache due to event network-changed-e66a5f7b-3d88-48a9-8467-e93315cc6f16. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1980.088478] env[62506]: DEBUG oslo_concurrency.lockutils [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] Acquiring lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1980.088625] env[62506]: DEBUG oslo_concurrency.lockutils [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] Acquired lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1980.088784] env[62506]: DEBUG nova.network.neutron [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Refreshing network info cache for port e66a5f7b-3d88-48a9-8467-e93315cc6f16 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1980.110689] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1980.146669] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1980.146870] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1980.147030] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1980.147220] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1980.147362] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1980.147502] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1980.147707] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1980.147859] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1980.148406] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1980.148716] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1980.148964] env[62506]: DEBUG nova.virt.hardware [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1980.149898] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4cdd0f0-7b20-4787-9054-85d589d64f99 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1980.159057] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1f5e880-718c-4008-a656-85ea5a4364b0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1980.369541] env[62506]: ERROR nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1980.369541] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1980.369541] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1980.369541] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1980.369541] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1980.369541] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1980.369541] env[62506]: ERROR nova.compute.manager raise self.value [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1980.369541] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1980.369541] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1980.369541] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1980.370104] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1980.370104] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1980.370104] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1980.370104] env[62506]: ERROR nova.compute.manager [ 1980.370223] env[62506]: Traceback (most recent call last): [ 1980.370223] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1980.370223] env[62506]: listener.cb(fileno) [ 1980.370223] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1980.370223] env[62506]: result = function(*args, **kwargs) [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1980.370223] env[62506]: return func(*args, **kwargs) [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1980.370223] env[62506]: raise e [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1980.370223] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1980.370223] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1980.370223] env[62506]: with excutils.save_and_reraise_exception(): [ 1980.370223] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1980.370223] env[62506]: self.force_reraise() [ 1980.370223] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1980.370223] env[62506]: raise self.value [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1980.370223] env[62506]: updated_port = self._update_port( [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1980.370223] env[62506]: _ensure_no_port_binding_failure(port) [ 1980.370223] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1980.370223] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1980.370223] env[62506]: nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1980.370223] env[62506]: Removing descriptor: 20 [ 1980.370955] env[62506]: ERROR nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Traceback (most recent call last): [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] yield resources [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self.driver.spawn(context, instance, image_meta, [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] vm_ref = self.build_virtual_machine(instance, [ 1980.370955] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] vif_infos = vmwarevif.get_vif_info(self._session, [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] for vif in network_info: [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return self._sync_wrapper(fn, *args, **kwargs) [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self.wait() [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self[:] = self._gt.wait() [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return self._exit_event.wait() [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1980.371365] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] result = hub.switch() [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return self.greenlet.switch() [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] result = function(*args, **kwargs) [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return func(*args, **kwargs) [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] raise e [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] nwinfo = self.network_api.allocate_for_instance( [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] created_port_ids = self._update_ports_for_instance( [ 1980.371774] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] with excutils.save_and_reraise_exception(): [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self.force_reraise() [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] raise self.value [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] updated_port = self._update_port( [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] _ensure_no_port_binding_failure(port) [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] raise exception.PortBindingFailed(port_id=port['id']) [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1980.372164] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] [ 1980.372507] env[62506]: INFO nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Terminating instance [ 1980.452327] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1980.478134] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1980.608288] env[62506]: DEBUG nova.network.neutron [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1980.617849] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1980.734645] env[62506]: DEBUG nova.network.neutron [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1980.781238] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de65300a-40e8-4bee-a92f-13a37fd87ad9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1980.789553] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e43f0234-ee31-4169-a473-d1ae98aeb651 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1980.819527] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52ae1551-b143-477f-8136-67b312817264 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1980.827155] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8fa5042-4b14-4eed-9222-8bf7f874337e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1980.840164] env[62506]: DEBUG nova.compute.provider_tree [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1980.875594] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquiring lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1981.122402] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Releasing lock "refresh_cache-3d034f40-5896-49bc-bed1-9062e1fde1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1981.123286] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1981.123640] env[62506]: DEBUG nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1981.126406] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1981.145089] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1981.239942] env[62506]: DEBUG oslo_concurrency.lockutils [req-855a2311-9d8e-44e3-99b2-d56a1b9f2108 req-742ea4ca-1906-4262-9fad-981704cd581c service nova] Releasing lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1981.240714] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquired lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1981.240714] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1981.342932] env[62506]: DEBUG nova.scheduler.client.report [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1981.610759] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1981.651030] env[62506]: DEBUG nova.network.neutron [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1981.765296] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1981.847711] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.886s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1981.848091] env[62506]: ERROR nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Traceback (most recent call last): [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self.driver.spawn(context, instance, image_meta, [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] vm_ref = self.build_virtual_machine(instance, [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] vif_infos = vmwarevif.get_vif_info(self._session, [ 1981.848091] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] for vif in network_info: [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] return self._sync_wrapper(fn, *args, **kwargs) [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self.wait() [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self[:] = self._gt.wait() [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] return self._exit_event.wait() [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] current.throw(*self._exc) [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1981.848420] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] result = function(*args, **kwargs) [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] return func(*args, **kwargs) [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] raise e [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] nwinfo = self.network_api.allocate_for_instance( [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] created_port_ids = self._update_ports_for_instance( [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] with excutils.save_and_reraise_exception(): [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] self.force_reraise() [ 1981.848771] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] raise self.value [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] updated_port = self._update_port( [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] _ensure_no_port_binding_failure(port) [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] raise exception.PortBindingFailed(port_id=port['id']) [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] nova.exception.PortBindingFailed: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. [ 1981.849157] env[62506]: ERROR nova.compute.manager [instance: b5d6b647-e24f-4e53-b448-0614334fd760] [ 1981.849157] env[62506]: DEBUG nova.compute.utils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1981.850243] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.956s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1981.852367] env[62506]: INFO nova.compute.claims [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1981.855650] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Build of instance b5d6b647-e24f-4e53-b448-0614334fd760 was re-scheduled: Binding failed for port 433f07a7-4387-40b5-91e7-a64a211149b0, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1981.856082] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1981.856309] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquiring lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1981.856460] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Acquired lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1981.856615] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1981.858293] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1982.117508] env[62506]: DEBUG nova.compute.manager [req-e84d0445-d7c4-4d0c-9651-9e62ec7cfa0b req-c7173fd8-253f-4670-96bb-ac05acd84b20 service nova] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Received event network-vif-deleted-e66a5f7b-3d88-48a9-8467-e93315cc6f16 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1982.155403] env[62506]: INFO nova.compute.manager [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] [instance: 3d034f40-5896-49bc-bed1-9062e1fde1e3] Took 1.03 seconds to deallocate network for instance. [ 1982.365975] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Releasing lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1982.366422] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1982.366619] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1982.367136] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a0c4b135-154c-4b00-9266-8727b5cda185 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1982.377284] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5962f194-f7dc-4bde-81fe-d76a0eb99695 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1982.387774] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1982.401843] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ecc8aab4-a172-4218-8b09-ccfde0d03903 could not be found. [ 1982.402058] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1982.402269] env[62506]: INFO nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1982.402511] env[62506]: DEBUG oslo.service.loopingcall [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1982.402733] env[62506]: DEBUG nova.compute.manager [-] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1982.402829] env[62506]: DEBUG nova.network.neutron [-] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1982.417441] env[62506]: DEBUG nova.network.neutron [-] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1982.469902] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1982.919761] env[62506]: DEBUG nova.network.neutron [-] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1982.974584] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Releasing lock "refresh_cache-b5d6b647-e24f-4e53-b448-0614334fd760" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1982.974818] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1982.974999] env[62506]: DEBUG nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1982.975185] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1982.993221] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1983.115941] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bee6928-c471-46da-9e7b-ec180476ab50 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1983.123468] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8eb21ae9-b8c7-4674-a908-ccd5508c79ba {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1983.154017] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48201901-b050-4f3c-9b01-0bd033d8d7ee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1983.161595] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b572ab18-ef0d-4452-aa48-dc51cf107338 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1983.176932] env[62506]: DEBUG nova.compute.provider_tree [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1983.184209] env[62506]: INFO nova.scheduler.client.report [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Deleted allocations for instance 3d034f40-5896-49bc-bed1-9062e1fde1e3 [ 1983.422409] env[62506]: INFO nova.compute.manager [-] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Took 1.02 seconds to deallocate network for instance. [ 1983.424746] env[62506]: DEBUG nova.compute.claims [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1983.424920] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1983.494973] env[62506]: DEBUG nova.network.neutron [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1983.610346] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1983.679989] env[62506]: DEBUG nova.scheduler.client.report [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1983.691658] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5133565e-9d17-45e7-bee2-201f1728308a tempest-InstanceActionsV221TestJSON-184957359 tempest-InstanceActionsV221TestJSON-184957359-project-member] Lock "3d034f40-5896-49bc-bed1-9062e1fde1e3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 122.860s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1983.997441] env[62506]: INFO nova.compute.manager [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] [instance: b5d6b647-e24f-4e53-b448-0614334fd760] Took 1.02 seconds to deallocate network for instance. [ 1984.185938] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.336s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1984.186510] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1984.189501] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.474s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1984.190845] env[62506]: INFO nova.compute.claims [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1984.193399] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1984.696021] env[62506]: DEBUG nova.compute.utils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1984.699828] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1984.700009] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1984.731637] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1984.768788] env[62506]: DEBUG nova.policy [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd3a51a6ee52e4771a5101819c51fad2d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '098afd27a2554b2b9183a48db3313906', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1985.032571] env[62506]: INFO nova.scheduler.client.report [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Deleted allocations for instance b5d6b647-e24f-4e53-b448-0614334fd760 [ 1985.201254] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1985.506221] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c34965be-3c1c-4dc4-875e-cbe6683b2163 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1985.514678] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e81d19-7793-4b46-be7b-f21ce9c3c996 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1985.552894] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d2902c29-6490-4993-abcc-e6f9455b727c tempest-ServerRescueTestJSONUnderV235-1069075862 tempest-ServerRescueTestJSONUnderV235-1069075862-project-member] Lock "b5d6b647-e24f-4e53-b448-0614334fd760" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 123.843s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1985.554116] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Successfully created port: 8332d8e8-3deb-48e7-8d41-d05f182c9d28 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1985.561022] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1094b6a2-1166-4a76-a5f6-21a73a6bcd62 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1985.566151] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a24061ea-6e25-41b0-a2ad-e025c8e3f52b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1985.580555] env[62506]: DEBUG nova.compute.provider_tree [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1986.060936] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1986.083732] env[62506]: DEBUG nova.scheduler.client.report [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1986.219065] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1986.245722] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1986.246459] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1986.246663] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1986.246957] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1986.247231] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1986.247489] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1986.247826] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1986.248123] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1986.248409] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1986.248691] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1986.248979] env[62506]: DEBUG nova.virt.hardware [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1986.250088] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31b579a4-3a9c-4402-86aa-3440d4e4bb0f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1986.258733] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d63fa14-e54c-4d60-8c44-d2440bf40cfb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1986.471616] env[62506]: DEBUG nova.compute.manager [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Received event network-changed-8332d8e8-3deb-48e7-8d41-d05f182c9d28 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1986.471616] env[62506]: DEBUG nova.compute.manager [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Refreshing instance network info cache due to event network-changed-8332d8e8-3deb-48e7-8d41-d05f182c9d28. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1986.471752] env[62506]: DEBUG oslo_concurrency.lockutils [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] Acquiring lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1986.472256] env[62506]: DEBUG oslo_concurrency.lockutils [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] Acquired lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1986.472256] env[62506]: DEBUG nova.network.neutron [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Refreshing network info cache for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1986.585287] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1986.589180] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.400s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1986.589655] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1986.596057] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.798s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1986.597841] env[62506]: INFO nova.compute.claims [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1986.646378] env[62506]: ERROR nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 1986.646378] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1986.646378] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1986.646378] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1986.646378] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1986.646378] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1986.646378] env[62506]: ERROR nova.compute.manager raise self.value [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1986.646378] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1986.646378] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1986.646378] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1986.646883] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1986.646883] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1986.646883] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 1986.646883] env[62506]: ERROR nova.compute.manager [ 1986.647636] env[62506]: Traceback (most recent call last): [ 1986.648282] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1986.648282] env[62506]: listener.cb(fileno) [ 1986.648282] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1986.648282] env[62506]: result = function(*args, **kwargs) [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1986.648282] env[62506]: return func(*args, **kwargs) [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1986.648282] env[62506]: raise e [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1986.648282] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1986.648282] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1986.648282] env[62506]: with excutils.save_and_reraise_exception(): [ 1986.648282] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1986.648282] env[62506]: self.force_reraise() [ 1986.648282] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1986.648282] env[62506]: raise self.value [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1986.648282] env[62506]: updated_port = self._update_port( [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1986.648282] env[62506]: _ensure_no_port_binding_failure(port) [ 1986.648282] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1986.648282] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1986.648282] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 1986.648282] env[62506]: Removing descriptor: 15 [ 1986.651019] env[62506]: ERROR nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Traceback (most recent call last): [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] yield resources [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self.driver.spawn(context, instance, image_meta, [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] vm_ref = self.build_virtual_machine(instance, [ 1986.651019] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] vif_infos = vmwarevif.get_vif_info(self._session, [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] for vif in network_info: [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return self._sync_wrapper(fn, *args, **kwargs) [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self.wait() [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self[:] = self._gt.wait() [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return self._exit_event.wait() [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1986.651431] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] result = hub.switch() [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return self.greenlet.switch() [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] result = function(*args, **kwargs) [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return func(*args, **kwargs) [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] raise e [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] nwinfo = self.network_api.allocate_for_instance( [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] created_port_ids = self._update_ports_for_instance( [ 1986.651843] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] with excutils.save_and_reraise_exception(): [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self.force_reraise() [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] raise self.value [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] updated_port = self._update_port( [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] _ensure_no_port_binding_failure(port) [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] raise exception.PortBindingFailed(port_id=port['id']) [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 1986.652236] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] [ 1986.653980] env[62506]: INFO nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Terminating instance [ 1987.002268] env[62506]: DEBUG nova.network.neutron [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1987.111909] env[62506]: DEBUG nova.compute.utils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1987.111909] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1987.111909] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1987.112675] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1987.151510] env[62506]: DEBUG nova.policy [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ab67b57f49ad47ea9aab58b4f7bc563f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c97e56471ed471aa4a0812ec92e082c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1987.154670] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquiring lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1987.186213] env[62506]: DEBUG nova.network.neutron [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1987.619084] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1987.664652] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Successfully created port: 3241c70c-7819-431a-93a5-b282f5712dcc {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1987.690409] env[62506]: DEBUG oslo_concurrency.lockutils [req-b125ee70-004a-43f8-9610-827175bf063d req-c390f88a-cb30-4088-a310-a49e65c1481e service nova] Releasing lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1987.690916] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquired lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1987.691095] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1988.008741] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76998c2a-2f39-4159-b113-6fd27aedca5d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.019103] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abf797ce-abee-440a-920d-a2cc3378fe2e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.051531] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c2ddadf-b570-4bab-a928-2c8a010a2848 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.062195] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4071bbc-3ced-4903-8291-5b7540ea441e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.073959] env[62506]: DEBUG nova.compute.provider_tree [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1988.218028] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1988.306977] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1988.496162] env[62506]: DEBUG nova.compute.manager [req-515681d3-d0e8-4cc1-9e2e-44c443f73c6b req-a08aca16-62f3-49fa-acb9-354b929ddf70 service nova] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Received event network-vif-deleted-8332d8e8-3deb-48e7-8d41-d05f182c9d28 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1988.505628] env[62506]: DEBUG nova.compute.manager [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Received event network-changed-3241c70c-7819-431a-93a5-b282f5712dcc {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1988.505833] env[62506]: DEBUG nova.compute.manager [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Refreshing instance network info cache due to event network-changed-3241c70c-7819-431a-93a5-b282f5712dcc. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1988.506068] env[62506]: DEBUG oslo_concurrency.lockutils [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] Acquiring lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1988.506274] env[62506]: DEBUG oslo_concurrency.lockutils [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] Acquired lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1988.506363] env[62506]: DEBUG nova.network.neutron [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Refreshing network info cache for port 3241c70c-7819-431a-93a5-b282f5712dcc {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1988.576139] env[62506]: DEBUG nova.scheduler.client.report [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1988.609943] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1988.632652] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1988.664027] env[62506]: ERROR nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 1988.664027] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1988.664027] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1988.664027] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1988.664027] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1988.664027] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1988.664027] env[62506]: ERROR nova.compute.manager raise self.value [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1988.664027] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1988.664027] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1988.664027] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1988.664687] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1988.664687] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1988.664687] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 1988.664687] env[62506]: ERROR nova.compute.manager [ 1988.664687] env[62506]: Traceback (most recent call last): [ 1988.664687] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1988.664687] env[62506]: listener.cb(fileno) [ 1988.664687] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1988.664687] env[62506]: result = function(*args, **kwargs) [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1988.664687] env[62506]: return func(*args, **kwargs) [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1988.664687] env[62506]: raise e [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1988.664687] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1988.664687] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1988.664687] env[62506]: with excutils.save_and_reraise_exception(): [ 1988.664687] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1988.664687] env[62506]: self.force_reraise() [ 1988.664687] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1988.664687] env[62506]: raise self.value [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1988.664687] env[62506]: updated_port = self._update_port( [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1988.664687] env[62506]: _ensure_no_port_binding_failure(port) [ 1988.664687] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1988.664687] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1988.665579] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 1988.665579] env[62506]: Removing descriptor: 20 [ 1988.667398] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1988.667594] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1988.667749] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1988.667930] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1988.668091] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1988.668242] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1988.670097] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1988.670183] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1988.670341] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1988.671054] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1988.671054] env[62506]: DEBUG nova.virt.hardware [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1988.671582] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15294cec-e07e-4c4d-88d1-7fcfa83ad27c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.680128] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f03048b-1536-4ca4-97ad-365996dde418 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.695713] env[62506]: ERROR nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] Traceback (most recent call last): [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] yield resources [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self.driver.spawn(context, instance, image_meta, [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] vm_ref = self.build_virtual_machine(instance, [ 1988.695713] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] vif_infos = vmwarevif.get_vif_info(self._session, [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] for vif in network_info: [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] return self._sync_wrapper(fn, *args, **kwargs) [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self.wait() [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self[:] = self._gt.wait() [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] return self._exit_event.wait() [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1988.696220] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] current.throw(*self._exc) [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] result = function(*args, **kwargs) [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] return func(*args, **kwargs) [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] raise e [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] nwinfo = self.network_api.allocate_for_instance( [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] created_port_ids = self._update_ports_for_instance( [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] with excutils.save_and_reraise_exception(): [ 1988.696625] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self.force_reraise() [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] raise self.value [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] updated_port = self._update_port( [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] _ensure_no_port_binding_failure(port) [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] raise exception.PortBindingFailed(port_id=port['id']) [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 1988.697410] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] [ 1988.697410] env[62506]: INFO nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Terminating instance [ 1988.809584] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Releasing lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1988.810058] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1988.810278] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1988.810591] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-928bb2c5-3f0e-4da0-9d32-b9eab329b8f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.820208] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11c51998-9696-4aba-98a8-90e9bc5ab66c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1988.842784] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 19afb920-58b0-4ad9-be99-d0a8e930c680 could not be found. [ 1988.843018] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1988.843204] env[62506]: INFO nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1988.843449] env[62506]: DEBUG oslo.service.loopingcall [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1988.843691] env[62506]: DEBUG nova.compute.manager [-] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1988.843762] env[62506]: DEBUG nova.network.neutron [-] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1988.862480] env[62506]: DEBUG nova.network.neutron [-] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1989.025658] env[62506]: DEBUG nova.network.neutron [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1989.084321] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.492s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1989.084841] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1989.087345] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.734s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1989.113101] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1989.115923] env[62506]: DEBUG nova.network.neutron [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1989.200118] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1989.365288] env[62506]: DEBUG nova.network.neutron [-] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1989.591573] env[62506]: DEBUG nova.compute.utils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1989.592952] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1989.593135] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1989.617409] env[62506]: DEBUG oslo_concurrency.lockutils [req-fd13c10d-3de5-42e3-97f4-7979947b0995 req-595b0276-d687-40d9-9c71-4d34a7946eea service nova] Releasing lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1989.620420] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1989.620608] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1989.671969] env[62506]: DEBUG nova.policy [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '520fe05ea52d4eea98ca08e4b1d73811', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '959b8b2dd1e047af96e3705eff02d631', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1989.863927] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f8b0cf8-cb24-42d3-aba9-11824d7c3d29 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1989.868101] env[62506]: INFO nova.compute.manager [-] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Took 1.02 seconds to deallocate network for instance. [ 1989.871890] env[62506]: DEBUG nova.compute.claims [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1989.872079] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1989.873067] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e373bdea-ebf5-4762-a630-afb35c51a67b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1989.903989] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e9d1efe-17d6-46a4-af9f-ba750d0dc8ae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1989.915239] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b2134b5-c57a-4099-9065-e361570643a5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1989.930761] env[62506]: DEBUG nova.compute.provider_tree [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1990.103873] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1990.140870] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1990.215857] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Successfully created port: 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1990.245077] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1990.436613] env[62506]: DEBUG nova.scheduler.client.report [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1990.537313] env[62506]: DEBUG nova.compute.manager [req-39cdaf58-e86d-47bd-a8af-32b8a43c0c75 req-a8fea91a-630a-4951-9845-f5ef87e14140 service nova] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Received event network-vif-deleted-3241c70c-7819-431a-93a5-b282f5712dcc {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1990.751071] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1990.751071] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1990.751071] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1990.751071] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8b655185-a2ab-4965-81a5-faa642a5167b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1990.757784] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec2dd1af-c073-4ac0-b649-9ffe75e35198 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1990.781614] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dc1c594f-7625-495a-8948-cee046bf204a could not be found. [ 1990.781992] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1990.783703] env[62506]: INFO nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1990.784144] env[62506]: DEBUG oslo.service.loopingcall [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1990.784509] env[62506]: DEBUG nova.compute.manager [-] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1990.786579] env[62506]: DEBUG nova.network.neutron [-] [instance: dc1c594f-7625-495a-8948-cee046bf204a] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1990.945018] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.855s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1990.945018] env[62506]: ERROR nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Traceback (most recent call last): [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self.driver.spawn(context, instance, image_meta, [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1990.945018] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] vm_ref = self.build_virtual_machine(instance, [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] vif_infos = vmwarevif.get_vif_info(self._session, [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] for vif in network_info: [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] return self._sync_wrapper(fn, *args, **kwargs) [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self.wait() [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self[:] = self._gt.wait() [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] return self._exit_event.wait() [ 1990.945395] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] current.throw(*self._exc) [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] result = function(*args, **kwargs) [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] return func(*args, **kwargs) [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] raise e [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] nwinfo = self.network_api.allocate_for_instance( [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] created_port_ids = self._update_ports_for_instance( [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1990.945809] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] with excutils.save_and_reraise_exception(): [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] self.force_reraise() [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] raise self.value [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] updated_port = self._update_port( [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] _ensure_no_port_binding_failure(port) [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] raise exception.PortBindingFailed(port_id=port['id']) [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] nova.exception.PortBindingFailed: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. [ 1990.946183] env[62506]: ERROR nova.compute.manager [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] [ 1990.946515] env[62506]: DEBUG nova.compute.utils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1990.950020] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Build of instance dcd5a800-1a28-45b3-8eff-a2e8cb871491 was re-scheduled: Binding failed for port d0b4e330-33dc-466c-8dcc-d5a60fefc322, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1990.950020] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1990.950020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquiring lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1990.950020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Acquired lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1990.950328] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1990.950328] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.464s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1990.970095] env[62506]: DEBUG nova.network.neutron [-] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1991.112394] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1991.142410] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1991.142736] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1991.142936] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1991.143177] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1991.143370] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1991.144194] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1991.144194] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1991.144194] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1991.144194] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1991.144389] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1991.144551] env[62506]: DEBUG nova.virt.hardware [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1991.145463] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a30f34ba-79e2-480e-9507-aef529c887c4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1991.154130] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06cbad1d-86fe-4313-a989-efe88502ca97 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1991.434980] env[62506]: ERROR nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 1991.434980] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1991.434980] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1991.434980] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1991.434980] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1991.434980] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1991.434980] env[62506]: ERROR nova.compute.manager raise self.value [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1991.434980] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1991.434980] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1991.434980] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1991.435531] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1991.435531] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1991.435531] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 1991.435531] env[62506]: ERROR nova.compute.manager [ 1991.435531] env[62506]: Traceback (most recent call last): [ 1991.435531] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1991.435531] env[62506]: listener.cb(fileno) [ 1991.435531] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1991.435531] env[62506]: result = function(*args, **kwargs) [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1991.435531] env[62506]: return func(*args, **kwargs) [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1991.435531] env[62506]: raise e [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1991.435531] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1991.435531] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1991.435531] env[62506]: with excutils.save_and_reraise_exception(): [ 1991.435531] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1991.435531] env[62506]: self.force_reraise() [ 1991.435531] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1991.435531] env[62506]: raise self.value [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1991.435531] env[62506]: updated_port = self._update_port( [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1991.435531] env[62506]: _ensure_no_port_binding_failure(port) [ 1991.435531] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1991.435531] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1991.436453] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 1991.436453] env[62506]: Removing descriptor: 20 [ 1991.436453] env[62506]: ERROR nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Traceback (most recent call last): [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] yield resources [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self.driver.spawn(context, instance, image_meta, [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1991.436453] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] vm_ref = self.build_virtual_machine(instance, [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] vif_infos = vmwarevif.get_vif_info(self._session, [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] for vif in network_info: [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return self._sync_wrapper(fn, *args, **kwargs) [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self.wait() [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self[:] = self._gt.wait() [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return self._exit_event.wait() [ 1991.436940] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] result = hub.switch() [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return self.greenlet.switch() [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] result = function(*args, **kwargs) [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return func(*args, **kwargs) [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] raise e [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] nwinfo = self.network_api.allocate_for_instance( [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1991.437330] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] created_port_ids = self._update_ports_for_instance( [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] with excutils.save_and_reraise_exception(): [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self.force_reraise() [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] raise self.value [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] updated_port = self._update_port( [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] _ensure_no_port_binding_failure(port) [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1991.437738] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] raise exception.PortBindingFailed(port_id=port['id']) [ 1991.438119] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 1991.438119] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] [ 1991.438119] env[62506]: INFO nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Terminating instance [ 1991.472235] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1991.474692] env[62506]: DEBUG nova.network.neutron [-] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1991.562027] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1991.729533] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8aacbb17-f3b5-497c-874e-1235bf701990 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1991.738017] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f2c704f-f7f5-43e5-893f-9881572111b3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1991.770279] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97f7b4fd-7973-4790-97ca-abf0e95ffffe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1991.778304] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4369fa8b-e163-4025-b53f-eff5ff626c13 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1991.792699] env[62506]: DEBUG nova.compute.provider_tree [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1991.940223] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1991.940484] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquired lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1991.940600] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1991.977475] env[62506]: INFO nova.compute.manager [-] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Took 1.19 seconds to deallocate network for instance. [ 1991.979841] env[62506]: DEBUG nova.compute.claims [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1991.979992] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1992.065621] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Releasing lock "refresh_cache-dcd5a800-1a28-45b3-8eff-a2e8cb871491" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1992.065621] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1992.065621] env[62506]: DEBUG nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1992.065780] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1992.080893] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1992.271993] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquiring lock "7f33e930-8252-4019-81e1-79329a1ee1e3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1992.272291] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Lock "7f33e930-8252-4019-81e1-79329a1ee1e3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1992.296179] env[62506]: DEBUG nova.scheduler.client.report [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1992.460702] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1992.513593] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1992.565964] env[62506]: DEBUG nova.compute.manager [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Received event network-changed-33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1992.565964] env[62506]: DEBUG nova.compute.manager [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Refreshing instance network info cache due to event network-changed-33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1992.565964] env[62506]: DEBUG oslo_concurrency.lockutils [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] Acquiring lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1992.583914] env[62506]: DEBUG nova.network.neutron [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1992.800471] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.851s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1992.801189] env[62506]: ERROR nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] Traceback (most recent call last): [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self.driver.spawn(context, instance, image_meta, [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] vm_ref = self.build_virtual_machine(instance, [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] vif_infos = vmwarevif.get_vif_info(self._session, [ 1992.801189] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] for vif in network_info: [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] return self._sync_wrapper(fn, *args, **kwargs) [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self.wait() [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self[:] = self._gt.wait() [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] return self._exit_event.wait() [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] current.throw(*self._exc) [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1992.801660] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] result = function(*args, **kwargs) [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] return func(*args, **kwargs) [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] raise e [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] nwinfo = self.network_api.allocate_for_instance( [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] created_port_ids = self._update_ports_for_instance( [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] with excutils.save_and_reraise_exception(): [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] self.force_reraise() [ 1992.802100] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] raise self.value [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] updated_port = self._update_port( [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] _ensure_no_port_binding_failure(port) [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] raise exception.PortBindingFailed(port_id=port['id']) [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] nova.exception.PortBindingFailed: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. [ 1992.802450] env[62506]: ERROR nova.compute.manager [instance: 04218b42-91dd-4045-8a40-f8829910e064] [ 1992.802450] env[62506]: DEBUG nova.compute.utils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1992.803040] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.186s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1992.804432] env[62506]: INFO nova.compute.claims [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1992.806873] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Build of instance 04218b42-91dd-4045-8a40-f8829910e064 was re-scheduled: Binding failed for port fd65fd9a-cb71-42a8-b33f-91460241309d, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1992.807289] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1992.807512] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1992.807658] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquired lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1992.807814] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1993.018078] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Releasing lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1993.018078] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1993.018078] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1993.018078] env[62506]: DEBUG oslo_concurrency.lockutils [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] Acquired lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1993.018321] env[62506]: DEBUG nova.network.neutron [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Refreshing network info cache for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1993.019305] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-331603e1-d1f7-48df-9d8d-2c274c11d694 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1993.028810] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bb4f43a-d61d-4092-992f-80d456524242 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1993.051213] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e3c3759c-81e6-415d-a5a5-b17cc16bdc2f could not be found. [ 1993.051433] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1993.051617] env[62506]: INFO nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1993.051844] env[62506]: DEBUG oslo.service.loopingcall [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1993.052053] env[62506]: DEBUG nova.compute.manager [-] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1993.052151] env[62506]: DEBUG nova.network.neutron [-] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1993.066487] env[62506]: DEBUG nova.network.neutron [-] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1993.085222] env[62506]: INFO nova.compute.manager [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] [instance: dcd5a800-1a28-45b3-8eff-a2e8cb871491] Took 1.02 seconds to deallocate network for instance. [ 1993.328664] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1993.402034] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1993.538395] env[62506]: DEBUG nova.network.neutron [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1993.568714] env[62506]: DEBUG nova.network.neutron [-] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1993.585515] env[62506]: DEBUG nova.network.neutron [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1993.905281] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Releasing lock "refresh_cache-04218b42-91dd-4045-8a40-f8829910e064" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1993.905554] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 1993.905698] env[62506]: DEBUG nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1993.905864] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1993.922324] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1994.040059] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a7e87af-a55a-425d-b708-1a25ec162821 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1994.048045] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67992b66-7a0c-4e3a-8e6c-e3402991ddb2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1994.077844] env[62506]: INFO nova.compute.manager [-] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Took 1.03 seconds to deallocate network for instance. [ 1994.080051] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7385d37-b874-4a7d-ac02-a59404b1d243 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1994.082698] env[62506]: DEBUG nova.compute.claims [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1994.082859] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1994.087225] env[62506]: DEBUG oslo_concurrency.lockutils [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] Releasing lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1994.087440] env[62506]: DEBUG nova.compute.manager [req-a6cfb20b-d6dc-48ff-8168-f04ce7f3b5d0 req-9201b0a3-f82c-4810-9bbb-fa8448826ad7 service nova] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Received event network-vif-deleted-33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1994.088728] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6eb82c2c-9122-43de-badc-b30c2c9027aa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1994.104587] env[62506]: DEBUG nova.compute.provider_tree [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1994.113809] env[62506]: INFO nova.scheduler.client.report [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Deleted allocations for instance dcd5a800-1a28-45b3-8eff-a2e8cb871491 [ 1994.425122] env[62506]: DEBUG nova.network.neutron [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1994.608094] env[62506]: DEBUG nova.scheduler.client.report [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1994.621508] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eda6ce9a-aebd-4116-b853-94da36d11784 tempest-ServersNegativeTestMultiTenantJSON-680844049 tempest-ServersNegativeTestMultiTenantJSON-680844049-project-member] Lock "dcd5a800-1a28-45b3-8eff-a2e8cb871491" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 130.975s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1994.927874] env[62506]: INFO nova.compute.manager [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: 04218b42-91dd-4045-8a40-f8829910e064] Took 1.02 seconds to deallocate network for instance. [ 1995.113509] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.310s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1995.114040] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1995.116388] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.664s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1995.117729] env[62506]: INFO nova.compute.claims [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1995.127035] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1995.621637] env[62506]: DEBUG nova.compute.utils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1995.627634] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1995.627634] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1995.648673] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1995.673905] env[62506]: DEBUG nova.policy [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3c9c372663b34fcaa0dac3e1c657e044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fc10dc6a19fb41cdbf046f6da6dc69fe', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1995.966303] env[62506]: INFO nova.scheduler.client.report [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Deleted allocations for instance 04218b42-91dd-4045-8a40-f8829910e064 [ 1995.987917] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Successfully created port: b28817ef-1c89-4e51-bfd5-5a39715f0ef3 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1996.125842] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1996.409917] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54574f13-81ec-417f-9fad-50b782ce9e91 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1996.417627] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9699f5f-e39f-4684-a1df-e088ecf85ab2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1996.449145] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4ff462f-e7cd-4d79-9f97-4164498cac7b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1996.456352] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3395aab1-0258-433f-99b9-52cc59513403 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1996.470478] env[62506]: DEBUG nova.compute.provider_tree [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1996.476627] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f98a86fe-64c7-4614-a84e-55ce892ca31a tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "04218b42-91dd-4045-8a40-f8829910e064" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 131.455s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1996.725372] env[62506]: DEBUG nova.compute.manager [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Received event network-changed-b28817ef-1c89-4e51-bfd5-5a39715f0ef3 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1996.726113] env[62506]: DEBUG nova.compute.manager [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Refreshing instance network info cache due to event network-changed-b28817ef-1c89-4e51-bfd5-5a39715f0ef3. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 1996.726113] env[62506]: DEBUG oslo_concurrency.lockutils [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] Acquiring lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1996.726113] env[62506]: DEBUG oslo_concurrency.lockutils [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] Acquired lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1996.726113] env[62506]: DEBUG nova.network.neutron [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Refreshing network info cache for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1996.974346] env[62506]: DEBUG nova.scheduler.client.report [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1996.978725] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1997.141405] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1997.157091] env[62506]: ERROR nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 1997.157091] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1997.157091] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1997.157091] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1997.157091] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1997.157091] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1997.157091] env[62506]: ERROR nova.compute.manager raise self.value [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1997.157091] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1997.157091] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1997.157091] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1997.157604] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1997.157604] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1997.157604] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 1997.157604] env[62506]: ERROR nova.compute.manager [ 1997.157604] env[62506]: Traceback (most recent call last): [ 1997.157604] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1997.157604] env[62506]: listener.cb(fileno) [ 1997.157604] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1997.157604] env[62506]: result = function(*args, **kwargs) [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1997.157604] env[62506]: return func(*args, **kwargs) [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1997.157604] env[62506]: raise e [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1997.157604] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1997.157604] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1997.157604] env[62506]: with excutils.save_and_reraise_exception(): [ 1997.157604] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1997.157604] env[62506]: self.force_reraise() [ 1997.157604] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1997.157604] env[62506]: raise self.value [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1997.157604] env[62506]: updated_port = self._update_port( [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1997.157604] env[62506]: _ensure_no_port_binding_failure(port) [ 1997.157604] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1997.157604] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1997.158540] env[62506]: nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 1997.158540] env[62506]: Removing descriptor: 20 [ 1997.170665] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1997.171114] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1997.171341] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1997.171547] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1997.171692] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1997.171838] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1997.172102] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1997.172432] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1997.172838] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1997.173062] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1997.173276] env[62506]: DEBUG nova.virt.hardware [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1997.174172] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f530a37a-8a67-4d49-a9fe-ed8b03bce5a7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1997.182912] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0090789-f8f8-45b0-a93a-7147812588c6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1997.197212] env[62506]: ERROR nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Traceback (most recent call last): [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] yield resources [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self.driver.spawn(context, instance, image_meta, [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] vm_ref = self.build_virtual_machine(instance, [ 1997.197212] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] vif_infos = vmwarevif.get_vif_info(self._session, [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] for vif in network_info: [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] return self._sync_wrapper(fn, *args, **kwargs) [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self.wait() [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self[:] = self._gt.wait() [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] return self._exit_event.wait() [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1997.197696] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] current.throw(*self._exc) [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] result = function(*args, **kwargs) [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] return func(*args, **kwargs) [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] raise e [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] nwinfo = self.network_api.allocate_for_instance( [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] created_port_ids = self._update_ports_for_instance( [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] with excutils.save_and_reraise_exception(): [ 1997.198150] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self.force_reraise() [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] raise self.value [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] updated_port = self._update_port( [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] _ensure_no_port_binding_failure(port) [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] raise exception.PortBindingFailed(port_id=port['id']) [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 1997.198578] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] [ 1997.198578] env[62506]: INFO nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Terminating instance [ 1997.254572] env[62506]: DEBUG nova.network.neutron [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1997.393522] env[62506]: DEBUG nova.network.neutron [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1997.482109] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.366s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1997.482684] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 1997.485193] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.060s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1997.520639] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1997.704931] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1997.896359] env[62506]: DEBUG oslo_concurrency.lockutils [req-296905f6-bea7-48c7-86ee-73ee4735bfc6 req-ff7f3667-0bf2-4b93-acd4-b832f206b58f service nova] Releasing lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1997.896749] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1997.896937] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1997.996229] env[62506]: DEBUG nova.compute.utils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1998.000715] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 1998.001038] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1998.077412] env[62506]: DEBUG nova.policy [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2ebe9e623b0449ed8e2c35015ce3ba91', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '67d0c6b27c8240bdb68912df66c6beb8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 1998.284561] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56ac18c2-6f48-4231-a2d9-8a0551e96dc9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1998.292520] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b688cde5-01d8-4b6e-99ba-87e9c7e77e8d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1998.324434] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0579f0ff-6da5-4806-83e7-a54c57f01953 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1998.332886] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0497f45-a145-45e4-96dd-c69e280cb799 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1998.348598] env[62506]: DEBUG nova.compute.provider_tree [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1998.429323] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1998.435907] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Successfully created port: da9712de-bd75-4ed5-b184-3a97678c04a0 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1998.502052] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 1998.655891] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1998.854482] env[62506]: DEBUG nova.scheduler.client.report [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 1998.958156] env[62506]: DEBUG nova.compute.manager [req-3381d440-6d57-43bd-8258-f422ad2bd35b req-a346208f-4b16-4c98-b6eb-8e6841f3191d service nova] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Received event network-vif-deleted-b28817ef-1c89-4e51-bfd5-5a39715f0ef3 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 1999.158183] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1999.158619] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 1999.158830] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1999.159155] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7e3deb67-74d8-4c72-aca6-fbb08dd00eea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.170080] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56d1adda-969a-4936-b8fa-2704ec98763c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.193627] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8f808943-6107-49d9-966f-c4e6ae0cdac4 could not be found. [ 1999.194131] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1999.194419] env[62506]: INFO nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1999.194703] env[62506]: DEBUG oslo.service.loopingcall [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1999.195131] env[62506]: DEBUG nova.compute.manager [-] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 1999.195289] env[62506]: DEBUG nova.network.neutron [-] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1999.222182] env[62506]: DEBUG nova.network.neutron [-] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1999.283163] env[62506]: ERROR nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 1999.283163] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1999.283163] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1999.283163] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1999.283163] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1999.283163] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1999.283163] env[62506]: ERROR nova.compute.manager raise self.value [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1999.283163] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 1999.283163] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1999.283163] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1999.284146] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1999.284146] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1999.284146] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 1999.284146] env[62506]: ERROR nova.compute.manager [ 1999.284146] env[62506]: Traceback (most recent call last): [ 1999.284146] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1999.284146] env[62506]: listener.cb(fileno) [ 1999.284146] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1999.284146] env[62506]: result = function(*args, **kwargs) [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1999.284146] env[62506]: return func(*args, **kwargs) [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1999.284146] env[62506]: raise e [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1999.284146] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1999.284146] env[62506]: created_port_ids = self._update_ports_for_instance( [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1999.284146] env[62506]: with excutils.save_and_reraise_exception(): [ 1999.284146] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1999.284146] env[62506]: self.force_reraise() [ 1999.284146] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1999.284146] env[62506]: raise self.value [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1999.284146] env[62506]: updated_port = self._update_port( [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1999.284146] env[62506]: _ensure_no_port_binding_failure(port) [ 1999.284146] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1999.284146] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 1999.285836] env[62506]: nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 1999.285836] env[62506]: Removing descriptor: 20 [ 1999.362788] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.875s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1999.362788] env[62506]: ERROR nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Traceback (most recent call last): [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self.driver.spawn(context, instance, image_meta, [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1999.362788] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] vm_ref = self.build_virtual_machine(instance, [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] vif_infos = vmwarevif.get_vif_info(self._session, [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] for vif in network_info: [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return self._sync_wrapper(fn, *args, **kwargs) [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self.wait() [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self[:] = self._gt.wait() [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return self._exit_event.wait() [ 1999.363091] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] result = hub.switch() [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return self.greenlet.switch() [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] result = function(*args, **kwargs) [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] return func(*args, **kwargs) [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] raise e [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] nwinfo = self.network_api.allocate_for_instance( [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1999.363441] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] created_port_ids = self._update_ports_for_instance( [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] with excutils.save_and_reraise_exception(): [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] self.force_reraise() [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] raise self.value [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] updated_port = self._update_port( [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] _ensure_no_port_binding_failure(port) [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1999.363812] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] raise exception.PortBindingFailed(port_id=port['id']) [ 1999.364156] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] nova.exception.PortBindingFailed: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. [ 1999.364156] env[62506]: ERROR nova.compute.manager [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] [ 1999.364156] env[62506]: DEBUG nova.compute.utils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1999.364837] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.634s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1999.370258] env[62506]: INFO nova.compute.claims [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1999.372311] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Build of instance ecc8aab4-a172-4218-8b09-ccfde0d03903 was re-scheduled: Binding failed for port e66a5f7b-3d88-48a9-8467-e93315cc6f16, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 1999.374190] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 1999.374190] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquiring lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1999.374190] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Acquired lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1999.374190] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1999.502912] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1999.503194] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1999.518319] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 1999.545825] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 1999.546079] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1999.546238] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 1999.546420] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1999.546563] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 1999.546706] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 1999.546913] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 1999.547083] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 1999.547254] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 1999.547441] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 1999.547639] env[62506]: DEBUG nova.virt.hardware [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 1999.548505] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5987c20b-0c87-4341-be0a-3e38cd90a381 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.557052] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b7b54ff-8991-4bfe-a2fd-c3606008789f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.571552] env[62506]: ERROR nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Traceback (most recent call last): [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] yield resources [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self.driver.spawn(context, instance, image_meta, [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] vm_ref = self.build_virtual_machine(instance, [ 1999.571552] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] for vif in network_info: [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] return self._sync_wrapper(fn, *args, **kwargs) [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self.wait() [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self[:] = self._gt.wait() [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] return self._exit_event.wait() [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1999.571976] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] current.throw(*self._exc) [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] result = function(*args, **kwargs) [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] return func(*args, **kwargs) [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] raise e [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] nwinfo = self.network_api.allocate_for_instance( [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] created_port_ids = self._update_ports_for_instance( [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] with excutils.save_and_reraise_exception(): [ 1999.572409] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self.force_reraise() [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] raise self.value [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] updated_port = self._update_port( [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] _ensure_no_port_binding_failure(port) [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] raise exception.PortBindingFailed(port_id=port['id']) [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 1999.572846] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] [ 1999.572846] env[62506]: INFO nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Terminating instance [ 1999.724953] env[62506]: DEBUG nova.network.neutron [-] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1999.893732] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1999.981909] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2000.078609] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquiring lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2000.078839] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquired lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2000.078980] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2000.228054] env[62506]: INFO nova.compute.manager [-] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Took 1.03 seconds to deallocate network for instance. [ 2000.230484] env[62506]: DEBUG nova.compute.claims [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2000.230661] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2000.484797] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Releasing lock "refresh_cache-ecc8aab4-a172-4218-8b09-ccfde0d03903" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2000.485046] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2000.485234] env[62506]: DEBUG nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2000.485400] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2000.502222] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2000.582470] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fa84df4-1227-4ea4-98d3-a65e67a1f72b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2000.591792] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bbcf72f-5b1c-4d66-9912-d6039a690007 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2000.621804] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2000.623966] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21e5983d-1151-45ea-b509-cfe5f77aeb9c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2000.630971] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-414b6cb8-66b5-4451-b60b-32e2759f8bea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2000.646020] env[62506]: DEBUG nova.compute.provider_tree [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2000.669586] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2000.987232] env[62506]: DEBUG nova.compute.manager [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Received event network-changed-da9712de-bd75-4ed5-b184-3a97678c04a0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2000.987443] env[62506]: DEBUG nova.compute.manager [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Refreshing instance network info cache due to event network-changed-da9712de-bd75-4ed5-b184-3a97678c04a0. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2000.987654] env[62506]: DEBUG oslo_concurrency.lockutils [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] Acquiring lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2001.006319] env[62506]: DEBUG nova.network.neutron [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2001.149275] env[62506]: DEBUG nova.scheduler.client.report [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2001.173065] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Releasing lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2001.173065] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2001.173065] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2001.173367] env[62506]: DEBUG oslo_concurrency.lockutils [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] Acquired lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2001.173653] env[62506]: DEBUG nova.network.neutron [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Refreshing network info cache for port da9712de-bd75-4ed5-b184-3a97678c04a0 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2001.174482] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1f0c3446-957b-45a6-87b4-8bd520b471c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2001.183564] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3962bcc7-fcea-4470-a7c1-6acaf166632b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2001.205606] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3 could not be found. [ 2001.205807] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2001.205985] env[62506]: INFO nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2001.206232] env[62506]: DEBUG oslo.service.loopingcall [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2001.206424] env[62506]: DEBUG nova.compute.manager [-] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2001.206518] env[62506]: DEBUG nova.network.neutron [-] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2001.220789] env[62506]: DEBUG nova.network.neutron [-] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2001.509260] env[62506]: INFO nova.compute.manager [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] [instance: ecc8aab4-a172-4218-8b09-ccfde0d03903] Took 1.02 seconds to deallocate network for instance. [ 2001.654936] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.290s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2001.655505] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2001.658537] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.073s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2001.659944] env[62506]: INFO nova.compute.claims [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2001.693177] env[62506]: DEBUG nova.network.neutron [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2001.722704] env[62506]: DEBUG nova.network.neutron [-] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2001.758802] env[62506]: DEBUG nova.network.neutron [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2002.164368] env[62506]: DEBUG nova.compute.utils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2002.167679] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2002.167778] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2002.208020] env[62506]: DEBUG nova.policy [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'afd20bbed4444b499aa9f45a161efb43', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c4cf59d7b3e1424dbaf57fc4f44a472b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2002.225934] env[62506]: INFO nova.compute.manager [-] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Took 1.02 seconds to deallocate network for instance. [ 2002.227661] env[62506]: DEBUG nova.compute.claims [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2002.227846] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2002.260981] env[62506]: DEBUG oslo_concurrency.lockutils [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] Releasing lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2002.261248] env[62506]: DEBUG nova.compute.manager [req-22eada76-8175-4549-a063-ab923f95dfbf req-ab92af3c-ce3c-4480-93e2-aeb14154d7ba service nova] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Received event network-vif-deleted-da9712de-bd75-4ed5-b184-3a97678c04a0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2002.492216] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Successfully created port: db8aa08c-2052-40b4-8b6b-be69336b035f {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2002.539206] env[62506]: INFO nova.scheduler.client.report [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Deleted allocations for instance ecc8aab4-a172-4218-8b09-ccfde0d03903 [ 2002.668486] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2002.899340] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64465fad-1e24-4b8e-96ef-9265c2bf1e15 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.905319] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-975ebce0-6085-47a2-bad1-67653ad8a97c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.934315] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43d660ed-d800-4c48-a656-04b5a72a963a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.941510] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3beeb6d5-0587-4ec0-bf2e-b90911cdf6db {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.954963] env[62506]: DEBUG nova.compute.provider_tree [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2003.050748] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2ef36eb7-2c1a-4189-99c5-142a762de040 tempest-ServerMetadataTestJSON-319280244 tempest-ServerMetadataTestJSON-319280244-project-member] Lock "ecc8aab4-a172-4218-8b09-ccfde0d03903" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 135.851s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2003.135141] env[62506]: DEBUG nova.compute.manager [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Received event network-changed-db8aa08c-2052-40b4-8b6b-be69336b035f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2003.135342] env[62506]: DEBUG nova.compute.manager [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Refreshing instance network info cache due to event network-changed-db8aa08c-2052-40b4-8b6b-be69336b035f. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2003.135563] env[62506]: DEBUG oslo_concurrency.lockutils [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] Acquiring lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2003.135705] env[62506]: DEBUG oslo_concurrency.lockutils [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] Acquired lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2003.135858] env[62506]: DEBUG nova.network.neutron [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Refreshing network info cache for port db8aa08c-2052-40b4-8b6b-be69336b035f {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2003.298737] env[62506]: ERROR nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2003.298737] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2003.298737] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2003.298737] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2003.298737] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2003.298737] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2003.298737] env[62506]: ERROR nova.compute.manager raise self.value [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2003.298737] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2003.298737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2003.298737] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2003.299310] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2003.299310] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2003.299310] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2003.299310] env[62506]: ERROR nova.compute.manager [ 2003.299310] env[62506]: Traceback (most recent call last): [ 2003.299310] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2003.299310] env[62506]: listener.cb(fileno) [ 2003.299310] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2003.299310] env[62506]: result = function(*args, **kwargs) [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2003.299310] env[62506]: return func(*args, **kwargs) [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2003.299310] env[62506]: raise e [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2003.299310] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2003.299310] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2003.299310] env[62506]: with excutils.save_and_reraise_exception(): [ 2003.299310] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2003.299310] env[62506]: self.force_reraise() [ 2003.299310] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2003.299310] env[62506]: raise self.value [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2003.299310] env[62506]: updated_port = self._update_port( [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2003.299310] env[62506]: _ensure_no_port_binding_failure(port) [ 2003.299310] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2003.299310] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2003.300222] env[62506]: nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2003.300222] env[62506]: Removing descriptor: 20 [ 2003.458511] env[62506]: DEBUG nova.scheduler.client.report [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2003.556512] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2003.656320] env[62506]: DEBUG nova.network.neutron [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2003.680160] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2003.712448] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2003.712692] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2003.712880] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2003.713138] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2003.713294] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2003.713481] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2003.713741] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2003.713913] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2003.714140] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2003.714336] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2003.714548] env[62506]: DEBUG nova.virt.hardware [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2003.715524] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3338c49-065e-4a96-867a-cfaca6acbbd0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.724994] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-730e14a2-fe4d-4189-aa3e-a74057548058 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.739395] env[62506]: ERROR nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Traceback (most recent call last): [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] yield resources [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self.driver.spawn(context, instance, image_meta, [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] vm_ref = self.build_virtual_machine(instance, [ 2003.739395] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] vif_infos = vmwarevif.get_vif_info(self._session, [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] for vif in network_info: [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] return self._sync_wrapper(fn, *args, **kwargs) [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self.wait() [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self[:] = self._gt.wait() [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] return self._exit_event.wait() [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2003.739807] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] current.throw(*self._exc) [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] result = function(*args, **kwargs) [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] return func(*args, **kwargs) [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] raise e [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] nwinfo = self.network_api.allocate_for_instance( [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] created_port_ids = self._update_ports_for_instance( [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] with excutils.save_and_reraise_exception(): [ 2003.740251] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self.force_reraise() [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] raise self.value [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] updated_port = self._update_port( [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] _ensure_no_port_binding_failure(port) [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] raise exception.PortBindingFailed(port_id=port['id']) [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2003.740723] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] [ 2003.740723] env[62506]: INFO nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Terminating instance [ 2003.741751] env[62506]: DEBUG nova.network.neutron [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2003.963171] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.304s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2003.963708] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2003.966676] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 14.854s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2003.966852] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2003.967009] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2003.967299] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.095s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2003.971227] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c36c466-e143-46d4-af10-dff226aad705 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.979884] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a00cbe6b-742b-436c-960c-c54d54a679d1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.994836] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81e33e09-7793-43c5-87c7-7fde964bbd67 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.001979] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8e35af1-4942-446b-b655-139f562d6c37 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.033083] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181262MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2004.033268] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2004.078874] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2004.263220] env[62506]: DEBUG oslo_concurrency.lockutils [req-c42fc171-bd45-4334-94eb-e04734de87cd req-eb6bf86e-8864-4af0-bcd5-dbf8415da981 service nova] Releasing lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2004.263838] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquiring lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2004.263998] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquired lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2004.264186] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2004.472484] env[62506]: DEBUG nova.compute.utils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2004.473899] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2004.474078] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2004.521083] env[62506]: DEBUG nova.policy [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5d39521abdff46e8986168d7088bafda', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '79de7ce7e96d40479ef9d398b194ff18', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2004.735396] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d691c4cf-8e31-4989-aaf1-f2bead28d117 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.743073] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78b855cb-e006-475a-b3a6-204bcaad8787 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.775908] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c7c013b-6725-4db3-96a2-4dff40043014 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.783865] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff405f1f-ef5f-4b33-8d4a-0d4cfa897641 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.798361] env[62506]: DEBUG nova.compute.provider_tree [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2004.800185] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2004.830188] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Successfully created port: 68cf1d1e-4ecc-4844-b081-68484f1dadd9 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2004.926617] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2004.978266] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2005.166339] env[62506]: DEBUG nova.compute.manager [req-72125bdd-0586-4921-ab22-e163aea6bde1 req-c3a409d8-6892-463a-8bb5-150fcd3a3b80 service nova] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Received event network-vif-deleted-db8aa08c-2052-40b4-8b6b-be69336b035f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2005.305540] env[62506]: DEBUG nova.scheduler.client.report [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2005.429889] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Releasing lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2005.431030] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2005.431210] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2005.431732] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1cb7a25a-efac-435c-a977-30dc4a27ee22 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.441496] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-402b3cca-0b48-456d-b143-7b2317b456e1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.466226] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c1943459-83bf-4461-80e0-d98da37b5d17 could not be found. [ 2005.466433] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2005.466628] env[62506]: INFO nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2005.466872] env[62506]: DEBUG oslo.service.loopingcall [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2005.467211] env[62506]: DEBUG nova.compute.manager [-] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2005.467312] env[62506]: DEBUG nova.network.neutron [-] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2005.488543] env[62506]: DEBUG nova.network.neutron [-] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2005.787810] env[62506]: ERROR nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2005.787810] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2005.787810] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2005.787810] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2005.787810] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2005.787810] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2005.787810] env[62506]: ERROR nova.compute.manager raise self.value [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2005.787810] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2005.787810] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2005.787810] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2005.788541] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2005.788541] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2005.788541] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2005.788541] env[62506]: ERROR nova.compute.manager [ 2005.788541] env[62506]: Traceback (most recent call last): [ 2005.788541] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2005.788541] env[62506]: listener.cb(fileno) [ 2005.788541] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2005.788541] env[62506]: result = function(*args, **kwargs) [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2005.788541] env[62506]: return func(*args, **kwargs) [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2005.788541] env[62506]: raise e [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2005.788541] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2005.788541] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2005.788541] env[62506]: with excutils.save_and_reraise_exception(): [ 2005.788541] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2005.788541] env[62506]: self.force_reraise() [ 2005.788541] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2005.788541] env[62506]: raise self.value [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2005.788541] env[62506]: updated_port = self._update_port( [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2005.788541] env[62506]: _ensure_no_port_binding_failure(port) [ 2005.788541] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2005.788541] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2005.789447] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2005.789447] env[62506]: Removing descriptor: 20 [ 2005.810472] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.843s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2005.811195] env[62506]: ERROR nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Traceback (most recent call last): [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self.driver.spawn(context, instance, image_meta, [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] vm_ref = self.build_virtual_machine(instance, [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] vif_infos = vmwarevif.get_vif_info(self._session, [ 2005.811195] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] for vif in network_info: [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return self._sync_wrapper(fn, *args, **kwargs) [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self.wait() [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self[:] = self._gt.wait() [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return self._exit_event.wait() [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] result = hub.switch() [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2005.811589] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return self.greenlet.switch() [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] result = function(*args, **kwargs) [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] return func(*args, **kwargs) [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] raise e [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] nwinfo = self.network_api.allocate_for_instance( [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] created_port_ids = self._update_ports_for_instance( [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] with excutils.save_and_reraise_exception(): [ 2005.812017] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] self.force_reraise() [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] raise self.value [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] updated_port = self._update_port( [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] _ensure_no_port_binding_failure(port) [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] raise exception.PortBindingFailed(port_id=port['id']) [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] nova.exception.PortBindingFailed: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. [ 2005.812503] env[62506]: ERROR nova.compute.manager [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] [ 2005.812865] env[62506]: DEBUG nova.compute.utils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2005.813217] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.833s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2005.815974] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Build of instance 19afb920-58b0-4ad9-be99-d0a8e930c680 was re-scheduled: Binding failed for port 8332d8e8-3deb-48e7-8d41-d05f182c9d28, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2005.816413] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2005.816651] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquiring lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2005.816781] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Acquired lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2005.816937] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2005.991360] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2005.993796] env[62506]: DEBUG nova.network.neutron [-] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2006.018903] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2006.019183] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2006.019393] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2006.019578] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2006.019727] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2006.019878] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2006.020106] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2006.020271] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2006.020437] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2006.020602] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2006.020770] env[62506]: DEBUG nova.virt.hardware [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2006.022229] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7ceee87-6de8-4196-90dc-2acffe4d03e0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.030038] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c9b36eb-69de-48d1-bf4e-e0221715c391 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.044872] env[62506]: ERROR nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Traceback (most recent call last): [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] yield resources [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self.driver.spawn(context, instance, image_meta, [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] vm_ref = self.build_virtual_machine(instance, [ 2006.044872] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] vif_infos = vmwarevif.get_vif_info(self._session, [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] for vif in network_info: [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] return self._sync_wrapper(fn, *args, **kwargs) [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self.wait() [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self[:] = self._gt.wait() [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] return self._exit_event.wait() [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2006.045341] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] current.throw(*self._exc) [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] result = function(*args, **kwargs) [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] return func(*args, **kwargs) [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] raise e [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] nwinfo = self.network_api.allocate_for_instance( [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] created_port_ids = self._update_ports_for_instance( [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] with excutils.save_and_reraise_exception(): [ 2006.045778] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self.force_reraise() [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] raise self.value [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] updated_port = self._update_port( [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] _ensure_no_port_binding_failure(port) [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] raise exception.PortBindingFailed(port_id=port['id']) [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2006.046338] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] [ 2006.046338] env[62506]: INFO nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Terminating instance [ 2006.340155] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2006.426762] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2006.496632] env[62506]: INFO nova.compute.manager [-] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Took 1.03 seconds to deallocate network for instance. [ 2006.498791] env[62506]: DEBUG nova.compute.claims [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2006.499043] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2006.543931] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ec5079b-7aed-4b92-9732-e594aa4594dc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.549120] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquiring lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2006.549120] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquired lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2006.549120] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2006.552395] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9b3b3a3-8073-47c7-a4ea-4860ffde5e98 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.582878] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c0961bd-ac9a-41f8-aaca-2c10870b59d7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.590193] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4d535ae-ea78-454e-9a75-dd36483cdfc4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.603304] env[62506]: DEBUG nova.compute.provider_tree [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2006.931062] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Releasing lock "refresh_cache-19afb920-58b0-4ad9-be99-d0a8e930c680" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2006.931062] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2006.931062] env[62506]: DEBUG nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2006.931638] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2006.947228] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2007.067620] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2007.106525] env[62506]: DEBUG nova.scheduler.client.report [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2007.146973] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2007.191166] env[62506]: DEBUG nova.compute.manager [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Received event network-changed-68cf1d1e-4ecc-4844-b081-68484f1dadd9 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2007.191364] env[62506]: DEBUG nova.compute.manager [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Refreshing instance network info cache due to event network-changed-68cf1d1e-4ecc-4844-b081-68484f1dadd9. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2007.191731] env[62506]: DEBUG oslo_concurrency.lockutils [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] Acquiring lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2007.450199] env[62506]: DEBUG nova.network.neutron [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2007.613018] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.799s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2007.613386] env[62506]: ERROR nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] Traceback (most recent call last): [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self.driver.spawn(context, instance, image_meta, [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] vm_ref = self.build_virtual_machine(instance, [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] vif_infos = vmwarevif.get_vif_info(self._session, [ 2007.613386] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] for vif in network_info: [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] return self._sync_wrapper(fn, *args, **kwargs) [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self.wait() [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self[:] = self._gt.wait() [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] return self._exit_event.wait() [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] current.throw(*self._exc) [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2007.613868] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] result = function(*args, **kwargs) [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] return func(*args, **kwargs) [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] raise e [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] nwinfo = self.network_api.allocate_for_instance( [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] created_port_ids = self._update_ports_for_instance( [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] with excutils.save_and_reraise_exception(): [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] self.force_reraise() [ 2007.614353] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] raise self.value [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] updated_port = self._update_port( [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] _ensure_no_port_binding_failure(port) [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] raise exception.PortBindingFailed(port_id=port['id']) [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] nova.exception.PortBindingFailed: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. [ 2007.614774] env[62506]: ERROR nova.compute.manager [instance: dc1c594f-7625-495a-8948-cee046bf204a] [ 2007.614774] env[62506]: DEBUG nova.compute.utils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2007.615618] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.533s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2007.618368] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Build of instance dc1c594f-7625-495a-8948-cee046bf204a was re-scheduled: Binding failed for port 3241c70c-7819-431a-93a5-b282f5712dcc, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2007.618759] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2007.618977] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2007.619151] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2007.619330] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2007.649266] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Releasing lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2007.649743] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2007.649937] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2007.650245] env[62506]: DEBUG oslo_concurrency.lockutils [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] Acquired lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2007.650411] env[62506]: DEBUG nova.network.neutron [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Refreshing network info cache for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2007.651370] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f4627675-49ed-4756-8670-7567a5600ca8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2007.661364] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ad52251-69d7-40a9-b0ae-2b0b6a116840 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2007.683190] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756 could not be found. [ 2007.683409] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2007.683586] env[62506]: INFO nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2007.683819] env[62506]: DEBUG oslo.service.loopingcall [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2007.684048] env[62506]: DEBUG nova.compute.manager [-] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2007.684144] env[62506]: DEBUG nova.network.neutron [-] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2007.699265] env[62506]: DEBUG nova.network.neutron [-] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2007.954067] env[62506]: INFO nova.compute.manager [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] [instance: 19afb920-58b0-4ad9-be99-d0a8e930c680] Took 1.02 seconds to deallocate network for instance. [ 2008.138591] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2008.177381] env[62506]: DEBUG nova.network.neutron [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2008.201043] env[62506]: DEBUG nova.network.neutron [-] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2008.217072] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2008.255065] env[62506]: DEBUG nova.network.neutron [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2008.328630] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63fe7af1-839b-463d-8cb4-4b649df2bd70 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2008.336477] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3977742-6032-444c-9531-43aad25c8c86 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2008.366326] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34964c15-e472-460a-b881-5d3af256ea69 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2008.373292] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47d35df3-380a-4017-b4c0-09e3f66f74a7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2008.386008] env[62506]: DEBUG nova.compute.provider_tree [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2008.705897] env[62506]: INFO nova.compute.manager [-] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Took 1.02 seconds to deallocate network for instance. [ 2008.708706] env[62506]: DEBUG nova.compute.claims [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2008.708887] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2008.719531] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-dc1c594f-7625-495a-8948-cee046bf204a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2008.719740] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2008.719922] env[62506]: DEBUG nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2008.720549] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2008.737892] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2008.757817] env[62506]: DEBUG oslo_concurrency.lockutils [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] Releasing lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2008.759021] env[62506]: DEBUG nova.compute.manager [req-44e79178-3192-4654-abcf-bab4b435c12d req-207a7c8c-7448-4391-8ffe-e9ee3f9bbf88 service nova] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Received event network-vif-deleted-68cf1d1e-4ecc-4844-b081-68484f1dadd9 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2008.889022] env[62506]: DEBUG nova.scheduler.client.report [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2008.986564] env[62506]: INFO nova.scheduler.client.report [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Deleted allocations for instance 19afb920-58b0-4ad9-be99-d0a8e930c680 [ 2009.241913] env[62506]: DEBUG nova.network.neutron [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2009.393707] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.778s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2009.394400] env[62506]: ERROR nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Traceback (most recent call last): [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self.driver.spawn(context, instance, image_meta, [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] vm_ref = self.build_virtual_machine(instance, [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] vif_infos = vmwarevif.get_vif_info(self._session, [ 2009.394400] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] for vif in network_info: [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return self._sync_wrapper(fn, *args, **kwargs) [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self.wait() [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self[:] = self._gt.wait() [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return self._exit_event.wait() [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] result = hub.switch() [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2009.394781] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return self.greenlet.switch() [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] result = function(*args, **kwargs) [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] return func(*args, **kwargs) [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] raise e [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] nwinfo = self.network_api.allocate_for_instance( [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] created_port_ids = self._update_ports_for_instance( [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] with excutils.save_and_reraise_exception(): [ 2009.395207] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] self.force_reraise() [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] raise self.value [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] updated_port = self._update_port( [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] _ensure_no_port_binding_failure(port) [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] raise exception.PortBindingFailed(port_id=port['id']) [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] nova.exception.PortBindingFailed: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. [ 2009.395629] env[62506]: ERROR nova.compute.manager [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] [ 2009.395982] env[62506]: DEBUG nova.compute.utils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2009.396239] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.748s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2009.397641] env[62506]: INFO nova.compute.claims [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2009.400155] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Build of instance e3c3759c-81e6-415d-a5a5-b17cc16bdc2f was re-scheduled: Binding failed for port 33ae667d-ac1a-4ea1-91a5-91d1b62bd2ed, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2009.400569] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2009.400794] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2009.400938] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquired lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2009.401111] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2009.495331] env[62506]: DEBUG oslo_concurrency.lockutils [None req-4a00a1d3-72ef-4c75-8ce0-c2b5e64fbe5b tempest-ServersTestFqdnHostnames-610243037 tempest-ServersTestFqdnHostnames-610243037-project-member] Lock "19afb920-58b0-4ad9-be99-d0a8e930c680" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 140.505s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2009.744765] env[62506]: INFO nova.compute.manager [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: dc1c594f-7625-495a-8948-cee046bf204a] Took 1.02 seconds to deallocate network for instance. [ 2009.933866] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2009.998057] env[62506]: DEBUG nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2010.016908] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2010.515209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2010.520950] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Releasing lock "refresh_cache-e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2010.521175] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2010.521340] env[62506]: DEBUG nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2010.521517] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2010.538776] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2010.623804] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88dce5b6-9e72-444b-b0c7-b4237a37ce11 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2010.631700] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e3ea1db-b982-4cc7-a424-a80c86937123 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2010.670697] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e1a1ab9-8c18-4bc1-8c45-0a344e1d538c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2010.678575] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8601845e-34b4-4a70-b527-e874ebc2385d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2010.692253] env[62506]: DEBUG nova.compute.provider_tree [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2010.776384] env[62506]: INFO nova.scheduler.client.report [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Deleted allocations for instance dc1c594f-7625-495a-8948-cee046bf204a [ 2011.042760] env[62506]: DEBUG nova.network.neutron [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2011.197752] env[62506]: DEBUG nova.scheduler.client.report [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2011.289590] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ad89d2d0-42a6-4de6-93bb-4c811d516107 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "dc1c594f-7625-495a-8948-cee046bf204a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 138.925s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2011.547698] env[62506]: INFO nova.compute.manager [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: e3c3759c-81e6-415d-a5a5-b17cc16bdc2f] Took 1.03 seconds to deallocate network for instance. [ 2011.702577] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.306s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2011.703347] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2011.706388] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.186s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2011.707649] env[62506]: INFO nova.compute.claims [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2011.792954] env[62506]: DEBUG nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2012.215158] env[62506]: DEBUG nova.compute.utils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2012.218594] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2012.218798] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2012.240811] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "56c46b60-5feb-4b7a-8c4c-b010f4cf7009" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2012.241079] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "56c46b60-5feb-4b7a-8c4c-b010f4cf7009" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2012.268516] env[62506]: DEBUG nova.policy [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8615160e68af4fcf9e9cd0d9f97a0b3e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0e4a3d4707b74087b11ee299cd02976c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2012.318496] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2012.575661] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Successfully created port: d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2012.580249] env[62506]: INFO nova.scheduler.client.report [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Deleted allocations for instance e3c3759c-81e6-415d-a5a5-b17cc16bdc2f [ 2012.719929] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2012.952514] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90c2e451-c3b6-47bc-a397-ea3ab773c99b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2012.961491] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60cd43f4-f373-4f2a-977d-cd32c7635450 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2012.994208] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d749b9b-ca91-42df-bbc9-31312407fc06 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2013.001461] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3be55c83-c475-4845-8bd8-65d040fc6d5a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2013.014549] env[62506]: DEBUG nova.compute.provider_tree [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2013.091521] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e2a30f83-0993-40cb-978c-2e15d31e6601 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "e3c3759c-81e6-415d-a5a5-b17cc16bdc2f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 138.589s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2013.521050] env[62506]: DEBUG nova.scheduler.client.report [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2013.594407] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2013.738026] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2013.776708] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2013.776708] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2013.776891] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2013.777049] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2013.777172] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2013.777357] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2013.778250] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2013.778250] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2013.778250] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2013.778250] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2013.778598] env[62506]: DEBUG nova.virt.hardware [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2013.779226] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14626b8c-42e2-4ed8-b87d-d6d50f20b076 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2013.788043] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65199110-2460-4115-a6bf-8c3f18f5d5d2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2013.853620] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "635dfc2e-d53a-467a-a23e-ee927ff5b2b1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2013.853825] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "635dfc2e-d53a-467a-a23e-ee927ff5b2b1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2013.873212] env[62506]: DEBUG nova.compute.manager [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Received event network-changed-d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2013.873212] env[62506]: DEBUG nova.compute.manager [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Refreshing instance network info cache due to event network-changed-d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2013.873363] env[62506]: DEBUG oslo_concurrency.lockutils [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] Acquiring lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2013.873513] env[62506]: DEBUG oslo_concurrency.lockutils [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] Acquired lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2013.873670] env[62506]: DEBUG nova.network.neutron [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Refreshing network info cache for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2014.028988] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.323s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2014.029554] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2014.032401] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.801s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2014.119744] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2014.142643] env[62506]: ERROR nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2014.142643] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2014.142643] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2014.142643] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2014.142643] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2014.142643] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2014.142643] env[62506]: ERROR nova.compute.manager raise self.value [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2014.142643] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2014.142643] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2014.142643] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2014.143255] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2014.143255] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2014.143255] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2014.143255] env[62506]: ERROR nova.compute.manager [ 2014.143255] env[62506]: Traceback (most recent call last): [ 2014.143255] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2014.143255] env[62506]: listener.cb(fileno) [ 2014.143255] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2014.143255] env[62506]: result = function(*args, **kwargs) [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2014.143255] env[62506]: return func(*args, **kwargs) [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2014.143255] env[62506]: raise e [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2014.143255] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2014.143255] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2014.143255] env[62506]: with excutils.save_and_reraise_exception(): [ 2014.143255] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2014.143255] env[62506]: self.force_reraise() [ 2014.143255] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2014.143255] env[62506]: raise self.value [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2014.143255] env[62506]: updated_port = self._update_port( [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2014.143255] env[62506]: _ensure_no_port_binding_failure(port) [ 2014.143255] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2014.143255] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2014.144230] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2014.144230] env[62506]: Removing descriptor: 15 [ 2014.144230] env[62506]: ERROR nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Traceback (most recent call last): [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] yield resources [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self.driver.spawn(context, instance, image_meta, [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2014.144230] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] vm_ref = self.build_virtual_machine(instance, [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] vif_infos = vmwarevif.get_vif_info(self._session, [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] for vif in network_info: [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return self._sync_wrapper(fn, *args, **kwargs) [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self.wait() [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self[:] = self._gt.wait() [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return self._exit_event.wait() [ 2014.144702] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] result = hub.switch() [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return self.greenlet.switch() [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] result = function(*args, **kwargs) [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return func(*args, **kwargs) [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] raise e [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] nwinfo = self.network_api.allocate_for_instance( [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2014.145116] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] created_port_ids = self._update_ports_for_instance( [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] with excutils.save_and_reraise_exception(): [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self.force_reraise() [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] raise self.value [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] updated_port = self._update_port( [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] _ensure_no_port_binding_failure(port) [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2014.145535] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] raise exception.PortBindingFailed(port_id=port['id']) [ 2014.145941] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2014.145941] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] [ 2014.145941] env[62506]: INFO nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Terminating instance [ 2014.395323] env[62506]: DEBUG nova.network.neutron [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2014.529709] env[62506]: DEBUG nova.network.neutron [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2014.546824] env[62506]: DEBUG nova.compute.utils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2014.548815] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2014.549578] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2014.580495] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "11614111-9445-4299-8626-d65d2038b0cd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2014.580730] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "11614111-9445-4299-8626-d65d2038b0cd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2014.619277] env[62506]: DEBUG nova.policy [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e508d2263c0c47d5bccfb1e2f1409316', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7d771ac74ac54fb38a84337eed72348a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2014.650129] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2014.784349] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd3c36f4-071e-4d82-9385-793acb22c6c9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2014.792160] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dd25a4a-b33c-4905-803c-18deff026377 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2014.822571] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bad10257-014b-4478-b4e7-cae80e2c0ed8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2014.830019] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78cf486c-7236-4e9a-a3f1-b3aa97c47910 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2014.843399] env[62506]: DEBUG nova.compute.provider_tree [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2014.886397] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "bd441dae-ac8b-41bd-8fae-15dca58f820e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2014.886641] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "bd441dae-ac8b-41bd-8fae-15dca58f820e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2014.966481] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Successfully created port: 9d37ddd3-1603-4d9a-93b1-859e8db24720 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2015.025547] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "b45f2334-c604-4454-993e-e1c16a6c1ef5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2015.025787] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "b45f2334-c604-4454-993e-e1c16a6c1ef5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2015.034077] env[62506]: DEBUG oslo_concurrency.lockutils [req-348c8f6b-4d55-4f15-865f-b85107cc64a0 req-5c5901ea-ee6a-464f-8e4f-a706cea33fc9 service nova] Releasing lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2015.034422] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2015.034603] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2015.053477] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2015.346764] env[62506]: DEBUG nova.scheduler.client.report [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2015.553056] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2015.645305] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2015.852662] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.820s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2015.853375] env[62506]: ERROR nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Traceback (most recent call last): [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self.driver.spawn(context, instance, image_meta, [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] vm_ref = self.build_virtual_machine(instance, [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] vif_infos = vmwarevif.get_vif_info(self._session, [ 2015.853375] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] for vif in network_info: [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] return self._sync_wrapper(fn, *args, **kwargs) [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self.wait() [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self[:] = self._gt.wait() [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] return self._exit_event.wait() [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] current.throw(*self._exc) [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2015.853818] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] result = function(*args, **kwargs) [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] return func(*args, **kwargs) [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] raise e [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] nwinfo = self.network_api.allocate_for_instance( [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] created_port_ids = self._update_ports_for_instance( [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] with excutils.save_and_reraise_exception(): [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] self.force_reraise() [ 2015.854277] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] raise self.value [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] updated_port = self._update_port( [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] _ensure_no_port_binding_failure(port) [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] raise exception.PortBindingFailed(port_id=port['id']) [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] nova.exception.PortBindingFailed: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. [ 2015.854704] env[62506]: ERROR nova.compute.manager [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] [ 2015.854704] env[62506]: DEBUG nova.compute.utils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2015.855338] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.627s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2015.858241] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Build of instance 8f808943-6107-49d9-966f-c4e6ae0cdac4 was re-scheduled: Binding failed for port b28817ef-1c89-4e51-bfd5-5a39715f0ef3, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2015.858658] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2015.858881] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2015.859037] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2015.859197] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2015.901602] env[62506]: DEBUG nova.compute.manager [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Received event network-vif-deleted-d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2015.901840] env[62506]: DEBUG nova.compute.manager [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Received event network-changed-9d37ddd3-1603-4d9a-93b1-859e8db24720 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2015.902013] env[62506]: DEBUG nova.compute.manager [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Refreshing instance network info cache due to event network-changed-9d37ddd3-1603-4d9a-93b1-859e8db24720. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2015.902226] env[62506]: DEBUG oslo_concurrency.lockutils [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] Acquiring lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2015.902365] env[62506]: DEBUG oslo_concurrency.lockutils [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] Acquired lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2015.902573] env[62506]: DEBUG nova.network.neutron [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Refreshing network info cache for port 9d37ddd3-1603-4d9a-93b1-859e8db24720 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2015.947084] env[62506]: ERROR nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2015.947084] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2015.947084] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2015.947084] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2015.947084] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2015.947084] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2015.947084] env[62506]: ERROR nova.compute.manager raise self.value [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2015.947084] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2015.947084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2015.947084] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2015.947608] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2015.947608] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2015.947608] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2015.947608] env[62506]: ERROR nova.compute.manager [ 2015.947608] env[62506]: Traceback (most recent call last): [ 2015.947608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2015.947608] env[62506]: listener.cb(fileno) [ 2015.947608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2015.947608] env[62506]: result = function(*args, **kwargs) [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2015.947608] env[62506]: return func(*args, **kwargs) [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2015.947608] env[62506]: raise e [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2015.947608] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2015.947608] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2015.947608] env[62506]: with excutils.save_and_reraise_exception(): [ 2015.947608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2015.947608] env[62506]: self.force_reraise() [ 2015.947608] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2015.947608] env[62506]: raise self.value [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2015.947608] env[62506]: updated_port = self._update_port( [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2015.947608] env[62506]: _ensure_no_port_binding_failure(port) [ 2015.947608] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2015.947608] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2015.948626] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2015.948626] env[62506]: Removing descriptor: 15 [ 2016.063106] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2016.089701] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2016.089965] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2016.090147] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2016.090339] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2016.090484] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2016.090634] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2016.090842] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2016.090996] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2016.091180] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2016.091341] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2016.091540] env[62506]: DEBUG nova.virt.hardware [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2016.092452] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-255a318f-5b19-4171-83a5-9c454a37d37d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.100461] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dc9ade3-ef8f-4110-8698-ba5bdfaa00a3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.114867] env[62506]: ERROR nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Traceback (most recent call last): [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] yield resources [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self.driver.spawn(context, instance, image_meta, [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] vm_ref = self.build_virtual_machine(instance, [ 2016.114867] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] vif_infos = vmwarevif.get_vif_info(self._session, [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] for vif in network_info: [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] return self._sync_wrapper(fn, *args, **kwargs) [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self.wait() [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self[:] = self._gt.wait() [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] return self._exit_event.wait() [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2016.115413] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] current.throw(*self._exc) [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] result = function(*args, **kwargs) [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] return func(*args, **kwargs) [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] raise e [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] nwinfo = self.network_api.allocate_for_instance( [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] created_port_ids = self._update_ports_for_instance( [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] with excutils.save_and_reraise_exception(): [ 2016.115903] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self.force_reraise() [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] raise self.value [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] updated_port = self._update_port( [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] _ensure_no_port_binding_failure(port) [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] raise exception.PortBindingFailed(port_id=port['id']) [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2016.116366] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] [ 2016.116366] env[62506]: INFO nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Terminating instance [ 2016.147367] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2016.147796] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2016.147994] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2016.148300] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6f2c2ccc-2369-471a-bf4f-14edf46f7ad5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.157413] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cbc6ff0-a6ec-4866-847c-6447c8f900a1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.180256] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 85f24cb7-a0c3-4598-9992-2726320ca76d could not be found. [ 2016.180694] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2016.180694] env[62506]: INFO nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2016.180866] env[62506]: DEBUG oslo.service.loopingcall [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2016.181086] env[62506]: DEBUG nova.compute.manager [-] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2016.181183] env[62506]: DEBUG nova.network.neutron [-] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2016.195163] env[62506]: DEBUG nova.network.neutron [-] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2016.379733] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2016.419895] env[62506]: DEBUG nova.network.neutron [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2016.451672] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2016.495442] env[62506]: DEBUG nova.network.neutron [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2016.590246] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a906285-e086-4a44-8c0f-c008dbde9e82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.597748] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03266d78-5cf0-461f-a4bf-99ddb20b2c2c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.627541] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2016.628468] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96f6b37b-9925-410b-b432-ca3fef73db74 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.636683] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b45225da-38f4-4b5d-8e90-19480ebb1568 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.649834] env[62506]: DEBUG nova.compute.provider_tree [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2016.697457] env[62506]: DEBUG nova.network.neutron [-] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2016.956037] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-8f808943-6107-49d9-966f-c4e6ae0cdac4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2016.956183] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2016.956292] env[62506]: DEBUG nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2016.956461] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2016.971715] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2016.999895] env[62506]: DEBUG oslo_concurrency.lockutils [req-487c8c1b-d13b-4307-849a-a359cf9d6c59 req-aac89791-bb78-4cbe-b51b-2eb70b8863c1 service nova] Releasing lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2017.000318] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2017.000470] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2017.153458] env[62506]: DEBUG nova.scheduler.client.report [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2017.199915] env[62506]: INFO nova.compute.manager [-] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Took 1.02 seconds to deallocate network for instance. [ 2017.202438] env[62506]: DEBUG nova.compute.claims [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2017.202753] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2017.474334] env[62506]: DEBUG nova.network.neutron [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2017.517931] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2017.595120] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2017.658441] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.803s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2017.659982] env[62506]: ERROR nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Traceback (most recent call last): [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self.driver.spawn(context, instance, image_meta, [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] vm_ref = self.build_virtual_machine(instance, [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 2017.659982] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] for vif in network_info: [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] return self._sync_wrapper(fn, *args, **kwargs) [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self.wait() [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self[:] = self._gt.wait() [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] return self._exit_event.wait() [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] current.throw(*self._exc) [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2017.660390] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] result = function(*args, **kwargs) [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] return func(*args, **kwargs) [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] raise e [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] nwinfo = self.network_api.allocate_for_instance( [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] created_port_ids = self._update_ports_for_instance( [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] with excutils.save_and_reraise_exception(): [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] self.force_reraise() [ 2017.660758] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] raise self.value [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] updated_port = self._update_port( [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] _ensure_no_port_binding_failure(port) [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] raise exception.PortBindingFailed(port_id=port['id']) [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] nova.exception.PortBindingFailed: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. [ 2017.661129] env[62506]: ERROR nova.compute.manager [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] [ 2017.661129] env[62506]: DEBUG nova.compute.utils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2017.661436] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Build of instance 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3 was re-scheduled: Binding failed for port da9712de-bd75-4ed5-b184-3a97678c04a0, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2017.661836] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2017.662070] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquiring lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2017.662220] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Acquired lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2017.662378] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2017.663345] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 13.630s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2017.926936] env[62506]: DEBUG nova.compute.manager [req-20cb0b63-69a0-40fd-af14-57b2aad06ed2 req-cc6bac07-6718-4f67-8d5c-5d48adf8bdf9 service nova] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Received event network-vif-deleted-9d37ddd3-1603-4d9a-93b1-859e8db24720 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2017.977191] env[62506]: INFO nova.compute.manager [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 8f808943-6107-49d9-966f-c4e6ae0cdac4] Took 1.02 seconds to deallocate network for instance. [ 2018.098593] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2018.099085] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2018.099310] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2018.099630] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ddcd600d-42d3-4de7-8e08-fb32037f50d7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2018.108104] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d545e7c-48c0-47a5-a2f6-4c4fb0a3f3da {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2018.130035] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3e30fb9a-092b-4886-b496-9a54cd4477d2 could not be found. [ 2018.130228] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2018.130406] env[62506]: INFO nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2018.130632] env[62506]: DEBUG oslo.service.loopingcall [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2018.130843] env[62506]: DEBUG nova.compute.manager [-] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2018.130938] env[62506]: DEBUG nova.network.neutron [-] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2018.145550] env[62506]: DEBUG nova.network.neutron [-] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2018.182336] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2018.229515] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2018.648051] env[62506]: DEBUG nova.network.neutron [-] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2018.732050] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Releasing lock "refresh_cache-7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2018.732269] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2018.732439] env[62506]: DEBUG nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2018.732696] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2018.748172] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2019.007035] env[62506]: INFO nova.scheduler.client.report [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Deleted allocations for instance 8f808943-6107-49d9-966f-c4e6ae0cdac4 [ 2019.150724] env[62506]: INFO nova.compute.manager [-] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Took 1.02 seconds to deallocate network for instance. [ 2019.153257] env[62506]: DEBUG nova.compute.claims [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2019.153442] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2019.194117] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8f808943-6107-49d9-966f-c4e6ae0cdac4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2019.251270] env[62506]: DEBUG nova.network.neutron [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2019.518897] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b22e57b9-b169-4184-8753-25c042c80605 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "8f808943-6107-49d9-966f-c4e6ae0cdac4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 135.524s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2019.697291] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2019.697624] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance c1943459-83bf-4461-80e0-d98da37b5d17 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2019.697707] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2019.697987] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 85f24cb7-a0c3-4598-9992-2726320ca76d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2019.698209] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 3e30fb9a-092b-4886-b496-9a54cd4477d2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2019.754565] env[62506]: INFO nova.compute.manager [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] [instance: 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3] Took 1.02 seconds to deallocate network for instance. [ 2020.022245] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2020.201760] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 1970c9fb-368f-4fb5-80be-1b8601124bfe has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2020.542503] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2020.705374] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 5e16bc57-424c-45b0-8e85-1d80bac92b18 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2020.783330] env[62506]: INFO nova.scheduler.client.report [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Deleted allocations for instance 7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3 [ 2021.052699] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "4ec5b60e-a35c-40b5-bf8b-167b9d856f98" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2021.052699] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "4ec5b60e-a35c-40b5-bf8b-167b9d856f98" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2021.209031] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8665bdf3-3930-4f45-b5be-e308f8f53828 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2021.293229] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b826412e-04c8-46d4-a0a2-56f7cb1521d0 tempest-ServerDiagnosticsNegativeTest-219220689 tempest-ServerDiagnosticsNegativeTest-219220689-project-member] Lock "7748d5e9-7aae-4b5d-80ac-b6a9f177d6b3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 123.955s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2021.712478] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dc0c0209-786b-44db-8f06-a429eec28bde has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2021.795322] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2022.214928] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 97091b4f-8448-49e5-9382-99e3358300c6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2022.321650] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2022.717807] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 50b8f972-e2bd-4526-ab28-bf887a240421 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2023.220580] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 73ff1d78-5841-4532-a3f0-51e3f8e3314e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2023.723387] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7f33e930-8252-4019-81e1-79329a1ee1e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2024.227858] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b9edd77e-2f5a-4050-aaeb-c32b67d9aef7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2024.730659] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 56c46b60-5feb-4b7a-8c4c-b010f4cf7009 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2025.233768] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 635dfc2e-d53a-467a-a23e-ee927ff5b2b1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2025.736976] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 11614111-9445-4299-8626-d65d2038b0cd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2026.240757] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance bd441dae-ac8b-41bd-8fae-15dca58f820e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2026.748451] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b45f2334-c604-4454-993e-e1c16a6c1ef5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2026.748892] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2026.748892] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1280MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2026.983066] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5883bef4-5263-4f3c-a38c-3f0df0d76882 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2026.989761] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7b5a6e4-cd6e-43a8-ba29-cba2d9772e3b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2027.019170] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb8d4153-19e4-4523-89ba-c60e6ff84aef {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2027.026731] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69387b41-da22-428a-8563-0a1442c53147 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2027.039830] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2027.125208] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquiring lock "94a23198-2d64-4003-93cc-93522fe70172" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2027.125445] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Lock "94a23198-2d64-4003-93cc-93522fe70172" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2027.544148] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2028.048494] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2028.048803] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 10.385s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2028.049042] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.970s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2028.050645] env[62506]: INFO nova.compute.claims [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2028.053319] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2028.053510] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11745}} [ 2028.558887] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] There are 3 instances to clean {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11754}} [ 2028.559071] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: b4bcab92-0007-4952-8d74-c470049edd71] Instance has had 0 of 5 cleanup attempts {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11758}} [ 2029.065187] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: af042cab-9132-4425-b017-133ec85afa0c] Instance has had 0 of 5 cleanup attempts {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11758}} [ 2029.287454] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2036b9d5-2162-4b44-8011-3ee7a25a1ce9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.295440] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27b6817f-bf92-44cf-88b0-3b4df97bd071 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.325599] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e8b4848-3a9a-4639-beec-27439d6fbdf9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.333237] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5c8d2c5-0f8b-4003-a269-1f22a6b3a2dd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2029.346162] env[62506]: DEBUG nova.compute.provider_tree [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2029.571045] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 3aca933d-f1e7-4dbe-a487-f8954da24981] Instance has had 0 of 5 cleanup attempts {{(pid=62506) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11758}} [ 2029.849852] env[62506]: DEBUG nova.scheduler.client.report [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2030.074155] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2030.074434] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Cleaning up deleted instances with incomplete migration {{(pid=62506) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11783}} [ 2030.354294] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.305s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2030.354822] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2030.357442] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.858s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2030.862201] env[62506]: DEBUG nova.compute.utils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2030.863612] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2030.863790] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2030.921489] env[62506]: DEBUG nova.policy [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '59f4559064a14dea9d74a4fef4eff6d7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '612309ae575f46bdb31861d769d09fd8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2031.102672] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-270e53bb-e405-4834-afba-766f6f09c0f3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2031.109987] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-184ebdf3-1d17-46ae-aeef-6c8da26ce10e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2031.139432] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6574a095-fdfb-49ec-a854-0eb68243ea7f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2031.146504] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef607fb2-4f1c-4a44-9e3b-a26ca9dda936 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2031.159203] env[62506]: DEBUG nova.compute.provider_tree [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2031.214243] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Successfully created port: 3c266e3b-afa5-4527-bb44-45483a7117ec {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2031.370218] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2031.577084] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2031.577360] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2031.577512] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 2031.577650] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 2031.661813] env[62506]: DEBUG nova.scheduler.client.report [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2031.836805] env[62506]: DEBUG nova.compute.manager [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Received event network-changed-3c266e3b-afa5-4527-bb44-45483a7117ec {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2031.836914] env[62506]: DEBUG nova.compute.manager [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Refreshing instance network info cache due to event network-changed-3c266e3b-afa5-4527-bb44-45483a7117ec. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2031.837152] env[62506]: DEBUG oslo_concurrency.lockutils [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] Acquiring lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2031.837270] env[62506]: DEBUG oslo_concurrency.lockutils [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] Acquired lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2031.837430] env[62506]: DEBUG nova.network.neutron [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Refreshing network info cache for port 3c266e3b-afa5-4527-bb44-45483a7117ec {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2032.006503] env[62506]: ERROR nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2032.006503] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2032.006503] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2032.006503] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2032.006503] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2032.006503] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2032.006503] env[62506]: ERROR nova.compute.manager raise self.value [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2032.006503] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2032.006503] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2032.006503] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2032.007089] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2032.007089] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2032.007089] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2032.007089] env[62506]: ERROR nova.compute.manager [ 2032.007089] env[62506]: Traceback (most recent call last): [ 2032.007089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2032.007089] env[62506]: listener.cb(fileno) [ 2032.007089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2032.007089] env[62506]: result = function(*args, **kwargs) [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2032.007089] env[62506]: return func(*args, **kwargs) [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2032.007089] env[62506]: raise e [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2032.007089] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2032.007089] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2032.007089] env[62506]: with excutils.save_and_reraise_exception(): [ 2032.007089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2032.007089] env[62506]: self.force_reraise() [ 2032.007089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2032.007089] env[62506]: raise self.value [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2032.007089] env[62506]: updated_port = self._update_port( [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2032.007089] env[62506]: _ensure_no_port_binding_failure(port) [ 2032.007089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2032.007089] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2032.008207] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2032.008207] env[62506]: Removing descriptor: 20 [ 2032.080779] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2032.081171] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2032.081171] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2032.081281] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2032.081328] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 2032.081514] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2032.081688] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2032.081858] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2032.082036] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2032.082181] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 2032.166814] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.809s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2032.167462] env[62506]: ERROR nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Traceback (most recent call last): [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self.driver.spawn(context, instance, image_meta, [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] vm_ref = self.build_virtual_machine(instance, [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] vif_infos = vmwarevif.get_vif_info(self._session, [ 2032.167462] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] for vif in network_info: [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] return self._sync_wrapper(fn, *args, **kwargs) [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self.wait() [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self[:] = self._gt.wait() [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] return self._exit_event.wait() [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] current.throw(*self._exc) [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2032.167768] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] result = function(*args, **kwargs) [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] return func(*args, **kwargs) [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] raise e [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] nwinfo = self.network_api.allocate_for_instance( [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] created_port_ids = self._update_ports_for_instance( [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] with excutils.save_and_reraise_exception(): [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] self.force_reraise() [ 2032.168099] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] raise self.value [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] updated_port = self._update_port( [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] _ensure_no_port_binding_failure(port) [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] raise exception.PortBindingFailed(port_id=port['id']) [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] nova.exception.PortBindingFailed: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. [ 2032.168418] env[62506]: ERROR nova.compute.manager [instance: c1943459-83bf-4461-80e0-d98da37b5d17] [ 2032.168418] env[62506]: DEBUG nova.compute.utils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2032.169462] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.461s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2032.172388] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Build of instance c1943459-83bf-4461-80e0-d98da37b5d17 was re-scheduled: Binding failed for port db8aa08c-2052-40b4-8b6b-be69336b035f, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2032.172884] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2032.173129] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquiring lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2032.173278] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Acquired lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2032.173438] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2032.354174] env[62506]: DEBUG nova.network.neutron [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2032.379874] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2032.403782] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2032.404036] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2032.404202] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2032.404382] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2032.404526] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2032.404667] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2032.404874] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2032.405043] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2032.405212] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2032.405373] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2032.405538] env[62506]: DEBUG nova.virt.hardware [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2032.406529] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-166a5a4c-9166-4e8b-a3c0-9b006fb8a11e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2032.415025] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14cda21a-fd30-4444-8653-a7fbf36a7747 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2032.428755] env[62506]: ERROR nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Traceback (most recent call last): [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] yield resources [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self.driver.spawn(context, instance, image_meta, [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] vm_ref = self.build_virtual_machine(instance, [ 2032.428755] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] vif_infos = vmwarevif.get_vif_info(self._session, [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] for vif in network_info: [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] return self._sync_wrapper(fn, *args, **kwargs) [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self.wait() [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self[:] = self._gt.wait() [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] return self._exit_event.wait() [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2032.429058] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] current.throw(*self._exc) [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] result = function(*args, **kwargs) [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] return func(*args, **kwargs) [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] raise e [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] nwinfo = self.network_api.allocate_for_instance( [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] created_port_ids = self._update_ports_for_instance( [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] with excutils.save_and_reraise_exception(): [ 2032.429461] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self.force_reraise() [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] raise self.value [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] updated_port = self._update_port( [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] _ensure_no_port_binding_failure(port) [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] raise exception.PortBindingFailed(port_id=port['id']) [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2032.429794] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] [ 2032.429794] env[62506]: INFO nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Terminating instance [ 2032.431085] env[62506]: DEBUG nova.network.neutron [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2032.697054] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2032.773657] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2032.891889] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cf41ad8-75f0-42e2-8afe-c3a0629bc4a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2032.899227] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e136f4c4-091d-4d22-8a7f-4dfbf6e59850 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2032.928488] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d7c46ac-3a0b-4cca-ad56-e8dc769792bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2032.935357] env[62506]: DEBUG oslo_concurrency.lockutils [req-ec764bbd-e89f-42ae-89ae-e8b792e93a2d req-745dc1c3-0085-4772-ba55-212485b8e36c service nova] Releasing lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2032.935986] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2032.936168] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquired lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2032.936340] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2032.938399] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ed0bdbc-24ac-4cea-a1fd-10191f404e34 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2032.953804] env[62506]: DEBUG nova.compute.provider_tree [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2033.277961] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Releasing lock "refresh_cache-c1943459-83bf-4461-80e0-d98da37b5d17" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2033.278235] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2033.278399] env[62506]: DEBUG nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2033.278567] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2033.296175] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2033.457258] env[62506]: DEBUG nova.scheduler.client.report [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2033.585518] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2033.654194] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2033.798356] env[62506]: DEBUG nova.network.neutron [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2033.859816] env[62506]: DEBUG nova.compute.manager [req-f2a301fc-63e5-4f3f-9288-fb15a566c6d9 req-8584725d-ad8c-44a7-b0e5-5330a1257473 service nova] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Received event network-vif-deleted-3c266e3b-afa5-4527-bb44-45483a7117ec {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2033.962918] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.793s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2033.963580] env[62506]: ERROR nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Traceback (most recent call last): [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self.driver.spawn(context, instance, image_meta, [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] vm_ref = self.build_virtual_machine(instance, [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] vif_infos = vmwarevif.get_vif_info(self._session, [ 2033.963580] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] for vif in network_info: [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] return self._sync_wrapper(fn, *args, **kwargs) [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self.wait() [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self[:] = self._gt.wait() [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] return self._exit_event.wait() [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] current.throw(*self._exc) [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2033.963873] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] result = function(*args, **kwargs) [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] return func(*args, **kwargs) [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] raise e [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] nwinfo = self.network_api.allocate_for_instance( [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] created_port_ids = self._update_ports_for_instance( [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] with excutils.save_and_reraise_exception(): [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] self.force_reraise() [ 2033.964202] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] raise self.value [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] updated_port = self._update_port( [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] _ensure_no_port_binding_failure(port) [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] raise exception.PortBindingFailed(port_id=port['id']) [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] nova.exception.PortBindingFailed: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. [ 2033.964529] env[62506]: ERROR nova.compute.manager [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] [ 2033.964529] env[62506]: DEBUG nova.compute.utils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2033.965400] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.450s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2033.966813] env[62506]: INFO nova.compute.claims [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2034.736465] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Build of instance 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756 was re-scheduled: Binding failed for port 68cf1d1e-4ecc-4844-b081-68484f1dadd9, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2034.736465] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2034.736465] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquiring lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2034.736907] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Acquired lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2034.736907] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2034.738129] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Releasing lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2034.741017] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2034.741017] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2034.741017] env[62506]: INFO nova.compute.manager [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] [instance: c1943459-83bf-4461-80e0-d98da37b5d17] Took 1.46 seconds to deallocate network for instance. [ 2034.742196] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-51f39a23-7291-4cb3-8192-f29307a6aa6f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2034.752329] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e86a204-b923-45f1-81fd-ed9d19d5c29d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2034.774365] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1970c9fb-368f-4fb5-80be-1b8601124bfe could not be found. [ 2034.774568] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2034.774742] env[62506]: INFO nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2034.774971] env[62506]: DEBUG oslo.service.loopingcall [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2034.775194] env[62506]: DEBUG nova.compute.manager [-] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2034.775284] env[62506]: DEBUG nova.network.neutron [-] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2034.789993] env[62506]: DEBUG nova.network.neutron [-] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2035.264158] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2035.293310] env[62506]: DEBUG nova.network.neutron [-] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2035.349643] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2035.773894] env[62506]: INFO nova.scheduler.client.report [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Deleted allocations for instance c1943459-83bf-4461-80e0-d98da37b5d17 [ 2035.795397] env[62506]: INFO nova.compute.manager [-] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Took 1.02 seconds to deallocate network for instance. [ 2035.797419] env[62506]: DEBUG nova.compute.claims [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2035.797571] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2035.854098] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Releasing lock "refresh_cache-0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2035.854387] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2035.854623] env[62506]: DEBUG nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2035.854830] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2035.869683] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2035.977723] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83f80ed5-063e-4db7-88cc-d7b1f14b09f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2035.985446] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7687bac6-523a-4cbc-a5cb-06a482ccdde9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2036.015328] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db70cbd6-ec9c-48b5-82f5-4038b4617af4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2036.022492] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba059190-2748-46ef-bd10-9c13ca134648 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2036.035201] env[62506]: DEBUG nova.compute.provider_tree [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2036.283936] env[62506]: DEBUG oslo_concurrency.lockutils [None req-21dd87c8-c2c8-4671-8487-4ad453c95fda tempest-ServersTestManualDisk-817825686 tempest-ServersTestManualDisk-817825686-project-member] Lock "c1943459-83bf-4461-80e0-d98da37b5d17" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 129.352s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2036.372137] env[62506]: DEBUG nova.network.neutron [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2036.538465] env[62506]: DEBUG nova.scheduler.client.report [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2036.787627] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2036.875490] env[62506]: INFO nova.compute.manager [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] [instance: 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756] Took 1.02 seconds to deallocate network for instance. [ 2037.044198] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.079s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2037.044745] env[62506]: DEBUG nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2037.047362] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.735s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2037.048714] env[62506]: INFO nova.compute.claims [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2037.310688] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2037.559137] env[62506]: DEBUG nova.compute.utils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2037.562904] env[62506]: DEBUG nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 2037.908560] env[62506]: INFO nova.scheduler.client.report [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Deleted allocations for instance 0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756 [ 2038.064483] env[62506]: DEBUG nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2038.312998] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03bba2f4-5ee7-4539-8a1c-28e046802489 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2038.321225] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4eec48f0-0316-4bb4-8314-2f23fe187e25 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2038.354168] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12780f8c-425e-4a0f-a214-2472b9432f56 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2038.362100] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ac2a56c-f436-4f06-b5e7-42a6cb717e1b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2038.381530] env[62506]: DEBUG nova.compute.provider_tree [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2038.417263] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5a9eb347-68bf-424b-a61c-791cb150ff7b tempest-AttachInterfacesUnderV243Test-67169722 tempest-AttachInterfacesUnderV243Test-67169722-project-member] Lock "0b7ecd6e-951b-4f5c-9c06-2a1b82cb9756" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 112.358s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2038.887404] env[62506]: DEBUG nova.scheduler.client.report [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2038.919972] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2039.076290] env[62506]: DEBUG nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2039.102955] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2039.103236] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2039.103474] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2039.104160] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2039.104160] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2039.104160] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2039.104160] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2039.104263] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2039.104420] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2039.104579] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2039.104748] env[62506]: DEBUG nova.virt.hardware [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2039.105646] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf7bffd6-4fd6-4bd0-b57f-19fb252a3a55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2039.113799] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-609f104a-0e16-45b0-9d45-096d1f4821aa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2039.127083] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2039.132533] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Creating folder: Project (a92eb10938a748298968e8922adc758c). Parent ref: group-v446953. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 2039.132814] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-7582634e-043a-4b0a-9e57-b24ea4a9ce58 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2039.143842] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Created folder: Project (a92eb10938a748298968e8922adc758c) in parent group-v446953. [ 2039.143968] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Creating folder: Instances. Parent ref: group-v446971. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 2039.144218] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-90cf6ecc-f2d8-454a-9214-5b778655437f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2039.153894] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Created folder: Instances in parent group-v446971. [ 2039.154143] env[62506]: DEBUG oslo.service.loopingcall [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2039.154310] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2039.154499] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2190e97b-5abd-46b6-9249-d2058aa5f32e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2039.170102] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2039.170102] env[62506]: value = "task-2190837" [ 2039.170102] env[62506]: _type = "Task" [ 2039.170102] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2039.177271] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190837, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2039.391025] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.343s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2039.391645] env[62506]: DEBUG nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2039.394616] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.275s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2039.397012] env[62506]: INFO nova.compute.claims [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2039.446379] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2039.679543] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190837, 'name': CreateVM_Task, 'duration_secs': 0.491562} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2039.679718] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2039.680159] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2039.680341] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2039.680760] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2039.681105] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-41d6ce6b-08fe-44e2-8520-e93c9d8f6a70 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2039.685908] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2039.685908] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52ce0f43-6db6-4348-7bed-1c3aea460cd4" [ 2039.685908] env[62506]: _type = "Task" [ 2039.685908] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2039.693960] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52ce0f43-6db6-4348-7bed-1c3aea460cd4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2039.901121] env[62506]: DEBUG nova.compute.utils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2039.902588] env[62506]: DEBUG nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 2040.196416] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52ce0f43-6db6-4348-7bed-1c3aea460cd4, 'name': SearchDatastore_Task, 'duration_secs': 0.025831} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2040.196751] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2040.197067] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2040.197523] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2040.197523] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2040.197632] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2040.197861] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f66ad357-0df6-4507-ac75-7f698562e284 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.207136] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2040.207330] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2040.208065] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-79d76c3f-6e3c-4381-b14e-18be031fb7b1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.213185] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2040.213185] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5247777b-cde7-b44d-82fe-a787e749cc41" [ 2040.213185] env[62506]: _type = "Task" [ 2040.213185] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2040.221724] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5247777b-cde7-b44d-82fe-a787e749cc41, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2040.405805] env[62506]: DEBUG nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2040.621784] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0306c5fd-a1c5-48ed-b727-93e94f4ade3c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.629508] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2deedb39-00a0-4acf-ab9e-34a1a481f7fe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.658748] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe5457b4-e93e-4e7a-94fa-a615863f5953 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.666139] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6c9ea64-6a13-4ac2-b1ee-807856a95dc6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.678952] env[62506]: DEBUG nova.compute.provider_tree [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2040.726443] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5247777b-cde7-b44d-82fe-a787e749cc41, 'name': SearchDatastore_Task, 'duration_secs': 0.008616} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2040.727249] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7e853c22-1e8f-49d2-843c-93c715023354 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2040.732602] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2040.732602] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52715128-f2ff-38a9-5365-cbd3b707f473" [ 2040.732602] env[62506]: _type = "Task" [ 2040.732602] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2040.740837] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52715128-f2ff-38a9-5365-cbd3b707f473, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2041.182562] env[62506]: DEBUG nova.scheduler.client.report [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2041.242941] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52715128-f2ff-38a9-5365-cbd3b707f473, 'name': SearchDatastore_Task, 'duration_secs': 0.009708} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2041.243275] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2041.243539] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 5e16bc57-424c-45b0-8e85-1d80bac92b18/5e16bc57-424c-45b0-8e85-1d80bac92b18.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2041.243808] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-2412272c-a0ca-4eee-bfb3-f16ff9189ce9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.250655] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2041.250655] env[62506]: value = "task-2190838" [ 2041.250655] env[62506]: _type = "Task" [ 2041.250655] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2041.258575] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190838, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2041.418055] env[62506]: DEBUG nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2041.445274] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2041.445568] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2041.445739] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2041.445922] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2041.446077] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2041.446223] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2041.446425] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2041.446587] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2041.446787] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2041.446967] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2041.447181] env[62506]: DEBUG nova.virt.hardware [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2041.448033] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adf2180d-cfa4-4e92-8e8e-ca7a99b22f08 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.456433] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-591f0788-325f-4422-8b97-7da753a0d556 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.470296] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2041.476295] env[62506]: DEBUG oslo.service.loopingcall [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2041.476517] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2041.476719] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9d7ee170-e7d0-4b36-8c6c-c585b574a074 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.494348] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2041.494348] env[62506]: value = "task-2190839" [ 2041.494348] env[62506]: _type = "Task" [ 2041.494348] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2041.501118] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2041.610944] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2041.690430] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.295s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2041.690430] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2041.693627] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 24.491s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2041.763171] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190838, 'name': CopyVirtualDisk_Task} progress is 25%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2042.004340] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task} progress is 99%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2042.198210] env[62506]: DEBUG nova.compute.utils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2042.205936] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2042.205936] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2042.247886] env[62506]: DEBUG nova.policy [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a2d3f8ecf879455e9a3c13c05ca64e52', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '68169a39b47a4c70b97872cc7270808f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2042.267609] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190838, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.679} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2042.267841] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 5e16bc57-424c-45b0-8e85-1d80bac92b18/5e16bc57-424c-45b0-8e85-1d80bac92b18.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2042.268103] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2042.268391] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-462eff03-f392-495f-8cd8-b2b0abcc0ff4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.276974] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2042.276974] env[62506]: value = "task-2190840" [ 2042.276974] env[62506]: _type = "Task" [ 2042.276974] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2042.292461] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190840, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2042.489893] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a65f374a-c90f-4c74-baf5-29aa4970f488 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.501570] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e27ffd8b-3902-40b6-83f4-0774a339a035 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.507930] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task} progress is 99%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2042.534642] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f8c313c-1270-4937-9b3c-51dfd3c0aa30 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.543587] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8aab5b9d-d4bf-4455-9d4d-687fff0a9b14 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.557640] env[62506]: DEBUG nova.compute.provider_tree [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2042.564647] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Successfully created port: 061c0690-c375-47ba-94a2-51828ae4c2a0 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2042.703964] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2042.788005] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190840, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.167256} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2042.788358] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2042.789103] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1684498a-791e-47c6-93c9-f19c304ec0b5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.814213] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Reconfiguring VM instance instance-00000033 to attach disk [datastore2] 5e16bc57-424c-45b0-8e85-1d80bac92b18/5e16bc57-424c-45b0-8e85-1d80bac92b18.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2042.815970] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-d550872a-e3fe-4a0c-89d8-c38bcdb41453 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.837200] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2042.837200] env[62506]: value = "task-2190841" [ 2042.837200] env[62506]: _type = "Task" [ 2042.837200] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2042.845580] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190841, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2043.006872] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task} progress is 99%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2043.063159] env[62506]: DEBUG nova.scheduler.client.report [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2043.213697] env[62506]: INFO nova.virt.block_device [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Booting with volume 70347a95-3831-48c1-90c2-a96826ddc52d at /dev/sda [ 2043.284835] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b05c57da-1478-4bdf-b8da-2f35b56f26fc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.297250] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47a57678-e8d1-47c0-b9a5-783f7e3c37ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.318385] env[62506]: DEBUG nova.compute.manager [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Received event network-changed-061c0690-c375-47ba-94a2-51828ae4c2a0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2043.319831] env[62506]: DEBUG nova.compute.manager [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Refreshing instance network info cache due to event network-changed-061c0690-c375-47ba-94a2-51828ae4c2a0. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2043.319831] env[62506]: DEBUG oslo_concurrency.lockutils [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] Acquiring lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2043.319831] env[62506]: DEBUG oslo_concurrency.lockutils [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] Acquired lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2043.319831] env[62506]: DEBUG nova.network.neutron [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Refreshing network info cache for port 061c0690-c375-47ba-94a2-51828ae4c2a0 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2043.341291] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-544e564f-cb2b-4f1a-a66e-78bb5cc0b350 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.350445] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190841, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2043.354402] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1575f13-b2f5-49a1-86db-b67698a97ec0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.378145] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17944194-5bcb-4673-a8c8-aef1efae0a24 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.385127] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70178a22-05ba-42c4-8fa7-dea7f4dcd8e4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.403130] env[62506]: DEBUG nova.virt.block_device [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Updating existing volume attachment record: 1ef0f599-92a3-49d9-9dd4-402130ac6aa4 {{(pid=62506) _volume_attach /opt/stack/nova/nova/virt/block_device.py:666}} [ 2043.505934] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task} progress is 99%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2043.511019] env[62506]: ERROR nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2043.511019] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2043.511019] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2043.511019] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2043.511019] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2043.511019] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2043.511019] env[62506]: ERROR nova.compute.manager raise self.value [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2043.511019] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2043.511019] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2043.511019] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2043.511552] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2043.511552] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2043.511552] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2043.511552] env[62506]: ERROR nova.compute.manager [ 2043.511552] env[62506]: Traceback (most recent call last): [ 2043.511552] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2043.511552] env[62506]: listener.cb(fileno) [ 2043.511552] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2043.511552] env[62506]: result = function(*args, **kwargs) [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2043.511552] env[62506]: return func(*args, **kwargs) [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2043.511552] env[62506]: raise e [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2043.511552] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2043.511552] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2043.511552] env[62506]: with excutils.save_and_reraise_exception(): [ 2043.511552] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2043.511552] env[62506]: self.force_reraise() [ 2043.511552] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2043.511552] env[62506]: raise self.value [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2043.511552] env[62506]: updated_port = self._update_port( [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2043.511552] env[62506]: _ensure_no_port_binding_failure(port) [ 2043.511552] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2043.511552] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2043.512403] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2043.512403] env[62506]: Removing descriptor: 20 [ 2043.570445] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.877s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2043.571150] env[62506]: ERROR nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Traceback (most recent call last): [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self.driver.spawn(context, instance, image_meta, [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] vm_ref = self.build_virtual_machine(instance, [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] vif_infos = vmwarevif.get_vif_info(self._session, [ 2043.571150] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] for vif in network_info: [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return self._sync_wrapper(fn, *args, **kwargs) [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self.wait() [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self[:] = self._gt.wait() [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return self._exit_event.wait() [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] result = hub.switch() [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2043.571413] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return self.greenlet.switch() [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] result = function(*args, **kwargs) [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] return func(*args, **kwargs) [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] raise e [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] nwinfo = self.network_api.allocate_for_instance( [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] created_port_ids = self._update_ports_for_instance( [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] with excutils.save_and_reraise_exception(): [ 2043.571675] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] self.force_reraise() [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] raise self.value [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] updated_port = self._update_port( [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] _ensure_no_port_binding_failure(port) [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] raise exception.PortBindingFailed(port_id=port['id']) [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] nova.exception.PortBindingFailed: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. [ 2043.571988] env[62506]: ERROR nova.compute.manager [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] [ 2043.572246] env[62506]: DEBUG nova.compute.utils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2043.574262] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 24.420s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2043.576505] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Build of instance 85f24cb7-a0c3-4598-9992-2726320ca76d was re-scheduled: Binding failed for port d28bfe6c-aad8-4f30-b39d-62d9fb6fd5a4, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2043.576948] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2043.577185] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2043.577336] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2043.577484] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2043.844955] env[62506]: DEBUG nova.network.neutron [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2043.856222] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190841, 'name': ReconfigVM_Task, 'duration_secs': 0.566197} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2043.856509] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Reconfigured VM instance instance-00000033 to attach disk [datastore2] 5e16bc57-424c-45b0-8e85-1d80bac92b18/5e16bc57-424c-45b0-8e85-1d80bac92b18.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2043.857253] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-5ac9b5fb-89e8-4a83-9e60-fbb21b1005d7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.864290] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2043.864290] env[62506]: value = "task-2190842" [ 2043.864290] env[62506]: _type = "Task" [ 2043.864290] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2043.875267] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190842, 'name': Rename_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2043.945792] env[62506]: DEBUG nova.network.neutron [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2044.006684] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task} progress is 99%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2044.096642] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2044.176202] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2044.330925] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1c62e03-ec06-443d-8073-9a1ddaac9d58 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2044.340139] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09a02b11-7554-4281-89ad-34f2a85023d6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2044.373963] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3be8ee5d-f86f-4dc2-b846-84ccb7be220b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2044.381357] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190842, 'name': Rename_Task, 'duration_secs': 0.231851} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2044.383476] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2044.383753] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-434d8a08-a99c-4ed9-b631-28e8bd6c2a81 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2044.386054] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19b94af9-c558-4e89-b0e3-5cb507fc223e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2044.399907] env[62506]: DEBUG nova.compute.provider_tree [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2044.402972] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2044.402972] env[62506]: value = "task-2190843" [ 2044.402972] env[62506]: _type = "Task" [ 2044.402972] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2044.410622] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190843, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2044.450122] env[62506]: DEBUG oslo_concurrency.lockutils [req-7e76449f-dab2-489c-bf8a-70236cb546e4 req-7664f30d-0b6b-481b-a979-e7cbdb39c8f3 service nova] Releasing lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2044.506567] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190839, 'name': CreateVM_Task, 'duration_secs': 2.630188} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2044.506741] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2044.507409] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2044.507409] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2044.507631] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2044.508216] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1ee742a9-c886-4ffa-88e9-a04e5377726e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2044.513043] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2044.513043] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52551427-e556-54df-b44c-26b54ecf931a" [ 2044.513043] env[62506]: _type = "Task" [ 2044.513043] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2044.519871] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52551427-e556-54df-b44c-26b54ecf931a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2044.679625] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-85f24cb7-a0c3-4598-9992-2726320ca76d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2044.679625] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2044.679625] env[62506]: DEBUG nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2044.679807] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2044.697893] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2044.810008] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquiring lock "caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2044.810265] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2044.903557] env[62506]: DEBUG nova.scheduler.client.report [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2044.917778] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190843, 'name': PowerOnVM_Task} progress is 89%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2045.023216] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52551427-e556-54df-b44c-26b54ecf931a, 'name': SearchDatastore_Task, 'duration_secs': 0.025117} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2045.023519] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2045.023753] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2045.023984] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2045.024149] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2045.024331] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2045.024606] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0dc98f6b-9926-4ed9-bea9-adc623ec5f34 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2045.033431] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2045.035292] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2045.035292] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-67b55bd1-89dc-43bf-9160-c56407983bf9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2045.039920] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2045.039920] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]521e9b6b-1bc5-2320-804a-b40abfda7bff" [ 2045.039920] env[62506]: _type = "Task" [ 2045.039920] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2045.047221] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]521e9b6b-1bc5-2320-804a-b40abfda7bff, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2045.200945] env[62506]: DEBUG nova.network.neutron [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2045.334652] env[62506]: DEBUG nova.compute.manager [req-af722a6c-993d-4108-b5b9-9426a1504253 req-8ad5178f-a3b7-488a-82f2-685b74e118c0 service nova] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Received event network-vif-deleted-061c0690-c375-47ba-94a2-51828ae4c2a0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2045.411658] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.838s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2045.412392] env[62506]: ERROR nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Traceback (most recent call last): [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self.driver.spawn(context, instance, image_meta, [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] vm_ref = self.build_virtual_machine(instance, [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] vif_infos = vmwarevif.get_vif_info(self._session, [ 2045.412392] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] for vif in network_info: [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] return self._sync_wrapper(fn, *args, **kwargs) [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self.wait() [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self[:] = self._gt.wait() [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] return self._exit_event.wait() [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] current.throw(*self._exc) [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2045.412665] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] result = function(*args, **kwargs) [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] return func(*args, **kwargs) [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] raise e [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] nwinfo = self.network_api.allocate_for_instance( [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] created_port_ids = self._update_ports_for_instance( [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] with excutils.save_and_reraise_exception(): [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] self.force_reraise() [ 2045.413031] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] raise self.value [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] updated_port = self._update_port( [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] _ensure_no_port_binding_failure(port) [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] raise exception.PortBindingFailed(port_id=port['id']) [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] nova.exception.PortBindingFailed: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. [ 2045.413409] env[62506]: ERROR nova.compute.manager [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] [ 2045.413409] env[62506]: DEBUG nova.compute.utils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2045.414454] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.872s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2045.415856] env[62506]: INFO nova.compute.claims [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2045.419055] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Build of instance 3e30fb9a-092b-4886-b496-9a54cd4477d2 was re-scheduled: Binding failed for port 9d37ddd3-1603-4d9a-93b1-859e8db24720, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2045.419229] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2045.419454] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2045.419598] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2045.419753] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2045.423679] env[62506]: DEBUG oslo_vmware.api [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190843, 'name': PowerOnVM_Task, 'duration_secs': 0.827523} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2045.424079] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2045.424275] env[62506]: INFO nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Took 6.35 seconds to spawn the instance on the hypervisor. [ 2045.424452] env[62506]: DEBUG nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2045.425203] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19ecd0b7-9bc3-468f-9b7d-dcce9dbae067 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2045.537303] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2045.537303] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2045.537303] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2045.537303] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2045.537484] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2045.537521] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2045.537683] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2045.537929] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2045.538132] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2045.538343] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2045.538535] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2045.538732] env[62506]: DEBUG nova.virt.hardware [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2045.540425] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-786dd0de-3bf4-487a-9fb0-cbdf8767c683 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2045.554857] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3f4d3d4-1996-42fa-9ee4-5e47ea6755e2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2045.558560] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]521e9b6b-1bc5-2320-804a-b40abfda7bff, 'name': SearchDatastore_Task, 'duration_secs': 0.007882} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2045.559676] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-21e91e81-7df4-40a2-a13a-d8e3735ed1dc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2045.569896] env[62506]: ERROR nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Traceback (most recent call last): [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] yield resources [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self.driver.spawn(context, instance, image_meta, [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] vm_ref = self.build_virtual_machine(instance, [ 2045.569896] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] vif_infos = vmwarevif.get_vif_info(self._session, [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] for vif in network_info: [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] return self._sync_wrapper(fn, *args, **kwargs) [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self.wait() [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self[:] = self._gt.wait() [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] return self._exit_event.wait() [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2045.570358] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] current.throw(*self._exc) [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] result = function(*args, **kwargs) [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] return func(*args, **kwargs) [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] raise e [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] nwinfo = self.network_api.allocate_for_instance( [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] created_port_ids = self._update_ports_for_instance( [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] with excutils.save_and_reraise_exception(): [ 2045.570647] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self.force_reraise() [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] raise self.value [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] updated_port = self._update_port( [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] _ensure_no_port_binding_failure(port) [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] raise exception.PortBindingFailed(port_id=port['id']) [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2045.570920] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] [ 2045.570920] env[62506]: INFO nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Terminating instance [ 2045.575246] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2045.575246] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52fcf824-e567-ae74-e6b9-be3553108d08" [ 2045.575246] env[62506]: _type = "Task" [ 2045.575246] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2045.583318] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52fcf824-e567-ae74-e6b9-be3553108d08, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2045.703605] env[62506]: INFO nova.compute.manager [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 85f24cb7-a0c3-4598-9992-2726320ca76d] Took 1.02 seconds to deallocate network for instance. [ 2045.941178] env[62506]: INFO nova.compute.manager [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Took 35.44 seconds to build instance. [ 2045.943930] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2046.025316] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2046.074482] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquiring lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2046.074684] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquired lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2046.074862] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2046.086911] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52fcf824-e567-ae74-e6b9-be3553108d08, 'name': SearchDatastore_Task, 'duration_secs': 0.027691} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2046.087597] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2046.087851] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2046.088182] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4e1e5439-af1e-4cd8-9184-6e9c23e0255a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2046.095988] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2046.095988] env[62506]: value = "task-2190844" [ 2046.095988] env[62506]: _type = "Task" [ 2046.095988] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2046.103889] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190844, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2046.443106] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6408d69c-55b4-4791-83cc-ae63e4b8faa4 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "5e16bc57-424c-45b0-8e85-1d80bac92b18" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 92.351s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2046.528454] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-3e30fb9a-092b-4886-b496-9a54cd4477d2" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2046.528723] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2046.528997] env[62506]: DEBUG nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2046.529259] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2046.546063] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2046.602731] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2046.610335] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190844, 'name': CopyVirtualDisk_Task} progress is 25%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2046.689034] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2046.731515] env[62506]: INFO nova.scheduler.client.report [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Deleted allocations for instance 85f24cb7-a0c3-4598-9992-2726320ca76d [ 2046.783867] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6da0b15b-009e-4092-a1b1-6cdfeeecd7fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2046.792769] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-305e051c-75e0-441f-8cdc-064d093191a4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2046.824429] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9d5692c-6c4a-4911-8dbf-c93bf003db10 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2046.832273] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ffd066a-c6dc-43c5-b9a4-999a50610581 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2046.845970] env[62506]: DEBUG nova.compute.provider_tree [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2046.945933] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2047.051355] env[62506]: DEBUG nova.network.neutron [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2047.106162] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190844, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.727124} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2047.106447] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2047.106647] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2047.106935] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-a5f1e823-b800-483e-b86d-08e566ad1529 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.113197] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2047.113197] env[62506]: value = "task-2190845" [ 2047.113197] env[62506]: _type = "Task" [ 2047.113197] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2047.120971] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190845, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2047.194896] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Releasing lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2047.195542] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2047.195856] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a2d57d76-9d5e-4a30-a66b-2d9443dc8124 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.205469] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7c87481-f1b2-4940-abd2-6d6e77a57609 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.228964] env[62506]: WARNING nova.virt.vmwareapi.driver [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance dc0c0209-786b-44db-8f06-a429eec28bde could not be found. [ 2047.229284] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2047.229583] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d0c73c08-0b0a-42b4-be7c-6131187e15db {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.237491] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-221f0769-21e7-4e26-9239-c67e4b5ceb66 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.247734] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ff21cd36-0f80-4935-967e-6dee3ab9802c tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "85f24cb7-a0c3-4598-9992-2726320ca76d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 120.523s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2047.261632] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dc0c0209-786b-44db-8f06-a429eec28bde could not be found. [ 2047.261862] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2047.262139] env[62506]: INFO nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Took 0.07 seconds to destroy the instance on the hypervisor. [ 2047.262383] env[62506]: DEBUG oslo.service.loopingcall [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2047.263947] env[62506]: DEBUG nova.compute.manager [-] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2047.263947] env[62506]: DEBUG nova.network.neutron [-] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2047.280297] env[62506]: DEBUG nova.network.neutron [-] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2047.349810] env[62506]: DEBUG nova.scheduler.client.report [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2047.466757] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2047.555198] env[62506]: INFO nova.compute.manager [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3e30fb9a-092b-4886-b496-9a54cd4477d2] Took 1.03 seconds to deallocate network for instance. [ 2047.624152] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190845, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.391793} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2047.624722] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2047.625500] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-415c3b89-b857-46fe-8ae4-10ec21b0efbd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.647200] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Reconfiguring VM instance instance-00000034 to attach disk [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2047.647451] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-f754ede0-8237-474f-9488-4d395c482a50 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2047.667605] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2047.667605] env[62506]: value = "task-2190846" [ 2047.667605] env[62506]: _type = "Task" [ 2047.667605] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2047.675347] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190846, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2047.750736] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2047.783289] env[62506]: DEBUG nova.network.neutron [-] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2047.856612] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.442s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2047.857199] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2047.859744] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.538s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2047.861151] env[62506]: INFO nova.compute.claims [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2048.177567] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190846, 'name': ReconfigVM_Task, 'duration_secs': 0.269673} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2048.177855] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Reconfigured VM instance instance-00000034 to attach disk [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2048.178474] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-eba76d74-ce8f-42aa-abc4-1ea2f053975d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2048.184890] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2048.184890] env[62506]: value = "task-2190847" [ 2048.184890] env[62506]: _type = "Task" [ 2048.184890] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2048.192946] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190847, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2048.268601] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2048.286160] env[62506]: INFO nova.compute.manager [-] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Took 1.02 seconds to deallocate network for instance. [ 2048.368147] env[62506]: DEBUG nova.compute.utils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2048.369914] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2048.369914] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2048.411252] env[62506]: DEBUG nova.policy [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a01d435093084caaa85feba239fcb70b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5c5301143ee049a9b5ccff80620d12ae', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2048.456810] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "6d89deb7-cc37-4feb-a998-fef1d8a92947" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2048.457070] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "6d89deb7-cc37-4feb-a998-fef1d8a92947" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2048.581959] env[62506]: INFO nova.scheduler.client.report [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Deleted allocations for instance 3e30fb9a-092b-4886-b496-9a54cd4477d2 [ 2048.609849] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2048.695155] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190847, 'name': Rename_Task, 'duration_secs': 0.229044} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2048.695486] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2048.695777] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-76490c6e-3a3a-4f12-8d15-5b687cf08794 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2048.702310] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2048.702310] env[62506]: value = "task-2190848" [ 2048.702310] env[62506]: _type = "Task" [ 2048.702310] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2048.711255] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190848, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2048.718064] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Successfully created port: 7affea15-4208-4851-91bd-9432223afd70 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2048.842836] env[62506]: INFO nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Took 0.56 seconds to detach 1 volumes for instance. [ 2048.845050] env[62506]: DEBUG nova.compute.claims [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2048.845232] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2048.873539] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2049.089263] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3128ada6-8d2c-4402-95c2-0cef189b0f7c tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "3e30fb9a-092b-4886-b496-9a54cd4477d2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 114.067s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2049.096612] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6dfc830-bd94-437b-9e21-f45f867a7ce3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.104872] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-416476d4-428f-45f7-ad42-57932dfe1c10 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.134524] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64797099-7af2-434d-9700-1280325becce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.145018] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b31ab72c-49bc-44e5-91ed-d62d2619446d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.156255] env[62506]: DEBUG nova.compute.provider_tree [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2049.211600] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190848, 'name': PowerOnVM_Task} progress is 89%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2049.385037] env[62506]: DEBUG nova.compute.manager [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Received event network-changed-7affea15-4208-4851-91bd-9432223afd70 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2049.385248] env[62506]: DEBUG nova.compute.manager [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Refreshing instance network info cache due to event network-changed-7affea15-4208-4851-91bd-9432223afd70. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2049.385468] env[62506]: DEBUG oslo_concurrency.lockutils [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] Acquiring lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2049.385611] env[62506]: DEBUG oslo_concurrency.lockutils [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] Acquired lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2049.385769] env[62506]: DEBUG nova.network.neutron [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Refreshing network info cache for port 7affea15-4208-4851-91bd-9432223afd70 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2049.562084] env[62506]: ERROR nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2049.562084] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2049.562084] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2049.562084] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2049.562084] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2049.562084] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2049.562084] env[62506]: ERROR nova.compute.manager raise self.value [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2049.562084] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2049.562084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2049.562084] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2049.562573] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2049.562573] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2049.562573] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2049.562573] env[62506]: ERROR nova.compute.manager [ 2049.562573] env[62506]: Traceback (most recent call last): [ 2049.562573] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2049.562573] env[62506]: listener.cb(fileno) [ 2049.562573] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2049.562573] env[62506]: result = function(*args, **kwargs) [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2049.562573] env[62506]: return func(*args, **kwargs) [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2049.562573] env[62506]: raise e [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2049.562573] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2049.562573] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2049.562573] env[62506]: with excutils.save_and_reraise_exception(): [ 2049.562573] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2049.562573] env[62506]: self.force_reraise() [ 2049.562573] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2049.562573] env[62506]: raise self.value [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2049.562573] env[62506]: updated_port = self._update_port( [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2049.562573] env[62506]: _ensure_no_port_binding_failure(port) [ 2049.562573] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2049.562573] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2049.563338] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2049.563338] env[62506]: Removing descriptor: 20 [ 2049.591737] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2049.610434] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2049.660604] env[62506]: DEBUG nova.scheduler.client.report [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2049.712546] env[62506]: DEBUG oslo_vmware.api [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190848, 'name': PowerOnVM_Task, 'duration_secs': 0.661918} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2049.712888] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2049.713155] env[62506]: INFO nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Took 8.29 seconds to spawn the instance on the hypervisor. [ 2049.713398] env[62506]: DEBUG nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2049.714295] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b1370b7-ca94-4097-bb5d-a27980477dc1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.885055] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2049.906024] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2049.906024] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2049.906024] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2049.906286] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2049.906286] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2049.906429] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2049.906635] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2049.906792] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2049.906956] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2049.907134] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2049.907309] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2049.908429] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c72ca105-e289-480f-b77a-2f5272a2b146 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.911596] env[62506]: DEBUG nova.network.neutron [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2049.918410] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26a2aba9-81b6-479c-85d0-1f462de9e683 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2049.932421] env[62506]: ERROR nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Traceback (most recent call last): [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] yield resources [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self.driver.spawn(context, instance, image_meta, [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] vm_ref = self.build_virtual_machine(instance, [ 2049.932421] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] vif_infos = vmwarevif.get_vif_info(self._session, [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] for vif in network_info: [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] return self._sync_wrapper(fn, *args, **kwargs) [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self.wait() [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self[:] = self._gt.wait() [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] return self._exit_event.wait() [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2049.932747] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] current.throw(*self._exc) [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] result = function(*args, **kwargs) [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] return func(*args, **kwargs) [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] raise e [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] nwinfo = self.network_api.allocate_for_instance( [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] created_port_ids = self._update_ports_for_instance( [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] with excutils.save_and_reraise_exception(): [ 2049.933161] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self.force_reraise() [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] raise self.value [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] updated_port = self._update_port( [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] _ensure_no_port_binding_failure(port) [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] raise exception.PortBindingFailed(port_id=port['id']) [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2049.933540] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] [ 2049.933540] env[62506]: INFO nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Terminating instance [ 2049.989682] env[62506]: DEBUG nova.network.neutron [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2050.113452] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2050.117758] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2050.166211] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.306s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2050.166815] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2050.169713] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.372s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2050.232123] env[62506]: INFO nova.compute.manager [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Took 37.93 seconds to build instance. [ 2050.437011] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2050.492189] env[62506]: DEBUG oslo_concurrency.lockutils [req-9fe00f80-f9ee-4ae6-b266-ede7ed2141cc req-0fffa33a-fa57-4ffe-83ad-156e8970b25a service nova] Releasing lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2050.492398] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2050.492596] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2050.673739] env[62506]: DEBUG nova.compute.utils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2050.675805] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2050.675805] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2050.727270] env[62506]: DEBUG nova.policy [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a01d435093084caaa85feba239fcb70b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5c5301143ee049a9b5ccff80620d12ae', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2050.733433] env[62506]: DEBUG oslo_concurrency.lockutils [None req-24bfaa42-4942-405d-82f6-dcb9a9f03759 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "8665bdf3-3930-4f45-b5be-e308f8f53828" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 96.447s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2050.945885] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1bf3fde1-0252-434c-b8d0-b5cb25d84bf7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2050.953842] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e889fcc4-59ce-4368-b1bb-471a4a737812 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2050.984792] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c414b6a-8943-4c41-abb3-7b3d8fab6e38 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2050.992573] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4846dbc3-5ce7-4bb6-ac1c-6ebbbc3bd270 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2050.997156] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Successfully created port: 624ec51d-a602-4a9a-aae1-0e1738502bf6 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2051.010165] env[62506]: DEBUG nova.compute.provider_tree [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2051.015966] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2051.064967] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "91925f54-176b-4f34-8634-46bf7cb73bdd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2051.065234] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "91925f54-176b-4f34-8634-46bf7cb73bdd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2051.113939] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2051.181409] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2051.235874] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2051.412470] env[62506]: INFO nova.compute.manager [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Rebuilding instance [ 2051.450503] env[62506]: DEBUG nova.compute.manager [req-31a7bc18-8342-4239-9187-5ad4e01e8f28 req-aaf25372-6b50-47f4-a7fe-0f83621eed4c service nova] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Received event network-vif-deleted-7affea15-4208-4851-91bd-9432223afd70 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2051.455386] env[62506]: DEBUG nova.compute.manager [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2051.456753] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fb91f6d-a1d6-4345-a0db-ea523d112b18 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2051.513826] env[62506]: DEBUG nova.scheduler.client.report [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2051.616066] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2051.616547] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2051.616750] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2051.617057] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-05334476-46b3-4b94-9d42-09931d8c35bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2051.626404] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cac8dfe4-c906-4c88-8df3-698ea55f89c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2051.650314] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 97091b4f-8448-49e5-9382-99e3358300c6 could not be found. [ 2051.650550] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2051.650733] env[62506]: INFO nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2051.650978] env[62506]: DEBUG oslo.service.loopingcall [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2051.651394] env[62506]: DEBUG nova.compute.manager [-] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2051.651494] env[62506]: DEBUG nova.network.neutron [-] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2051.665446] env[62506]: DEBUG nova.network.neutron [-] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2051.759908] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2051.933815] env[62506]: ERROR nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2051.933815] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2051.933815] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2051.933815] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2051.933815] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2051.933815] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2051.933815] env[62506]: ERROR nova.compute.manager raise self.value [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2051.933815] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2051.933815] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2051.933815] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2051.934301] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2051.934301] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2051.934301] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2051.934301] env[62506]: ERROR nova.compute.manager [ 2051.934301] env[62506]: Traceback (most recent call last): [ 2051.934301] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2051.934301] env[62506]: listener.cb(fileno) [ 2051.934301] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2051.934301] env[62506]: result = function(*args, **kwargs) [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2051.934301] env[62506]: return func(*args, **kwargs) [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2051.934301] env[62506]: raise e [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2051.934301] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2051.934301] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2051.934301] env[62506]: with excutils.save_and_reraise_exception(): [ 2051.934301] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2051.934301] env[62506]: self.force_reraise() [ 2051.934301] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2051.934301] env[62506]: raise self.value [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2051.934301] env[62506]: updated_port = self._update_port( [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2051.934301] env[62506]: _ensure_no_port_binding_failure(port) [ 2051.934301] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2051.934301] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2051.935097] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2051.935097] env[62506]: Removing descriptor: 20 [ 2052.020038] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.850s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2052.020723] env[62506]: ERROR nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Traceback (most recent call last): [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self.driver.spawn(context, instance, image_meta, [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] vm_ref = self.build_virtual_machine(instance, [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] vif_infos = vmwarevif.get_vif_info(self._session, [ 2052.020723] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] for vif in network_info: [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] return self._sync_wrapper(fn, *args, **kwargs) [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self.wait() [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self[:] = self._gt.wait() [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] return self._exit_event.wait() [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] current.throw(*self._exc) [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2052.021227] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] result = function(*args, **kwargs) [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] return func(*args, **kwargs) [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] raise e [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] nwinfo = self.network_api.allocate_for_instance( [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] created_port_ids = self._update_ports_for_instance( [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] with excutils.save_and_reraise_exception(): [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] self.force_reraise() [ 2052.021839] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] raise self.value [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] updated_port = self._update_port( [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] _ensure_no_port_binding_failure(port) [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] raise exception.PortBindingFailed(port_id=port['id']) [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] nova.exception.PortBindingFailed: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. [ 2052.022424] env[62506]: ERROR nova.compute.manager [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] [ 2052.022424] env[62506]: DEBUG nova.compute.utils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2052.022868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.712s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2052.024111] env[62506]: INFO nova.compute.claims [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2052.027776] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Build of instance 1970c9fb-368f-4fb5-80be-1b8601124bfe was re-scheduled: Binding failed for port 3c266e3b-afa5-4527-bb44-45483a7117ec, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2052.028369] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2052.028638] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2052.028826] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquired lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2052.028968] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2052.167627] env[62506]: DEBUG nova.network.neutron [-] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2052.190250] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2052.215714] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2052.216083] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2052.216294] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2052.216496] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2052.216646] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2052.216794] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2052.217081] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2052.217289] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2052.217468] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2052.217633] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2052.217835] env[62506]: DEBUG nova.virt.hardware [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2052.218741] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49a14164-b73b-424e-95d7-c80ccb3c6720 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2052.226825] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daebce20-f6c5-4785-9144-a8f55f58a6da {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2052.241017] env[62506]: ERROR nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Traceback (most recent call last): [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] yield resources [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self.driver.spawn(context, instance, image_meta, [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] vm_ref = self.build_virtual_machine(instance, [ 2052.241017] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] vif_infos = vmwarevif.get_vif_info(self._session, [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] for vif in network_info: [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] return self._sync_wrapper(fn, *args, **kwargs) [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self.wait() [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self[:] = self._gt.wait() [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] return self._exit_event.wait() [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2052.241317] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] current.throw(*self._exc) [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] result = function(*args, **kwargs) [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] return func(*args, **kwargs) [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] raise e [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] nwinfo = self.network_api.allocate_for_instance( [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] created_port_ids = self._update_ports_for_instance( [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] with excutils.save_and_reraise_exception(): [ 2052.241599] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self.force_reraise() [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] raise self.value [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] updated_port = self._update_port( [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] _ensure_no_port_binding_failure(port) [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] raise exception.PortBindingFailed(port_id=port['id']) [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2052.241895] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] [ 2052.241895] env[62506]: INFO nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Terminating instance [ 2052.471372] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2052.471694] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-0d3594ad-fdaa-4408-8820-2e9fe01df1c4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2052.480534] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2052.480534] env[62506]: value = "task-2190849" [ 2052.480534] env[62506]: _type = "Task" [ 2052.480534] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2052.488441] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190849, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2052.545960] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2052.628640] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2052.670384] env[62506]: INFO nova.compute.manager [-] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Took 1.02 seconds to deallocate network for instance. [ 2052.673132] env[62506]: DEBUG nova.compute.claims [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2052.673326] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2052.745672] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2052.745908] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2052.746309] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2052.990901] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190849, 'name': PowerOffVM_Task, 'duration_secs': 0.11037} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2052.991303] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2052.991436] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2052.992281] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b47e4d4b-3c0d-4a76-bfb3-ccb35dd4ed55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2052.999130] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2052.999384] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-2752aec2-4731-4ad7-a2c7-e1f84941485d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.024041] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2053.024254] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2053.024431] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleting the datastore file [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2053.024696] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-df5d7960-a142-4b29-8f38-ca5fbde75fcb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.033322] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2053.033322] env[62506]: value = "task-2190851" [ 2053.033322] env[62506]: _type = "Task" [ 2053.033322] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2053.041514] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190851, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2053.131298] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Releasing lock "refresh_cache-1970c9fb-368f-4fb5-80be-1b8601124bfe" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2053.131505] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2053.131695] env[62506]: DEBUG nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2053.131856] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2053.149892] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2053.254773] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c63416aa-bb1a-4d41-9c80-e34d7d7f5d87 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.262088] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e134954a-e734-4a38-b328-6a315d42526f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.265772] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2053.295346] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c7c0aa8-c45a-4208-ad70-02895963b387 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.302637] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08d65a85-2a63-4d3a-add8-05931d26aeed {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.316672] env[62506]: DEBUG nova.compute.provider_tree [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2053.351373] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2053.475219] env[62506]: DEBUG nova.compute.manager [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Received event network-changed-624ec51d-a602-4a9a-aae1-0e1738502bf6 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2053.475412] env[62506]: DEBUG nova.compute.manager [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Refreshing instance network info cache due to event network-changed-624ec51d-a602-4a9a-aae1-0e1738502bf6. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2053.475607] env[62506]: DEBUG oslo_concurrency.lockutils [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] Acquiring lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2053.547082] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190851, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.092827} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2053.547082] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2053.547082] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2053.547082] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2053.652400] env[62506]: DEBUG nova.network.neutron [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2053.821095] env[62506]: DEBUG nova.scheduler.client.report [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2053.853711] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2053.854247] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2053.854512] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2053.854857] env[62506]: DEBUG oslo_concurrency.lockutils [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] Acquired lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2053.855455] env[62506]: DEBUG nova.network.neutron [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Refreshing network info cache for port 624ec51d-a602-4a9a-aae1-0e1738502bf6 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2053.856145] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ae737bfa-da4d-418f-9322-64f65e8cef7a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.865952] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dff26558-c4e3-41f0-b40e-01a4a02e4c23 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.890802] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 50b8f972-e2bd-4526-ab28-bf887a240421 could not be found. [ 2053.890802] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2053.890802] env[62506]: INFO nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2053.890802] env[62506]: DEBUG oslo.service.loopingcall [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2053.890802] env[62506]: DEBUG nova.compute.manager [-] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2053.890802] env[62506]: DEBUG nova.network.neutron [-] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2053.904152] env[62506]: DEBUG nova.network.neutron [-] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2054.154840] env[62506]: INFO nova.compute.manager [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 1970c9fb-368f-4fb5-80be-1b8601124bfe] Took 1.02 seconds to deallocate network for instance. [ 2054.325901] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.303s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2054.326499] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2054.329113] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.883s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2054.330916] env[62506]: INFO nova.compute.claims [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2054.375639] env[62506]: DEBUG nova.network.neutron [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2054.406423] env[62506]: DEBUG nova.network.neutron [-] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2054.445214] env[62506]: DEBUG nova.network.neutron [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2054.576631] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2054.576853] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2054.577052] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2054.577292] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2054.577450] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2054.577597] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2054.577803] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2054.577964] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2054.578255] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2054.578329] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2054.578473] env[62506]: DEBUG nova.virt.hardware [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2054.579334] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5da47ba8-4c72-486e-8afd-745bfc17cad3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2054.587758] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14b1939d-b283-40f7-9497-91cfcc9eccd5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2054.600931] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2054.606369] env[62506]: DEBUG oslo.service.loopingcall [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2054.606589] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2054.606791] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-afded433-91e1-4f29-a049-c9912ad9daaa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2054.624777] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2054.624777] env[62506]: value = "task-2190852" [ 2054.624777] env[62506]: _type = "Task" [ 2054.624777] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2054.632457] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190852, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2054.835919] env[62506]: DEBUG nova.compute.utils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2054.839499] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2054.839674] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2054.878514] env[62506]: DEBUG nova.policy [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b37c0529800a43eca96641d95d60394f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '68001a3c57354f5ab7b9010a90ac0c92', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2054.909268] env[62506]: INFO nova.compute.manager [-] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Took 1.02 seconds to deallocate network for instance. [ 2054.911530] env[62506]: DEBUG nova.compute.claims [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2054.911721] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2054.947749] env[62506]: DEBUG oslo_concurrency.lockutils [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] Releasing lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2054.948041] env[62506]: DEBUG nova.compute.manager [req-9a5c0593-2630-46cf-bcda-e2eaf10e8d7d req-89066c09-ac24-4df6-9319-308702c6e266 service nova] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Received event network-vif-deleted-624ec51d-a602-4a9a-aae1-0e1738502bf6 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2055.135176] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190852, 'name': CreateVM_Task, 'duration_secs': 0.311271} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2055.135363] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2055.135753] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2055.135913] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2055.136269] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2055.136553] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-52f0bab5-4862-4394-aa8a-fda3f2849a9c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.141176] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2055.141176] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]526c9792-ac7a-4b83-c2d2-8918347d58cf" [ 2055.141176] env[62506]: _type = "Task" [ 2055.141176] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2055.149669] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]526c9792-ac7a-4b83-c2d2-8918347d58cf, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2055.150543] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Successfully created port: d68a073f-0677-4771-bf93-5df437d2690a {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2055.185476] env[62506]: INFO nova.scheduler.client.report [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Deleted allocations for instance 1970c9fb-368f-4fb5-80be-1b8601124bfe [ 2055.340160] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2055.592165] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba455a6a-b134-467e-b90d-ab967899132e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.599455] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91b896c2-1da3-4e9c-8971-ea5e8ef68562 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.639890] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-060c22df-41bd-433c-94aa-e90039f9ba55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.651242] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]526c9792-ac7a-4b83-c2d2-8918347d58cf, 'name': SearchDatastore_Task, 'duration_secs': 0.013455} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2055.653529] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2055.653752] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2055.653887] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2055.654047] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2055.654236] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2055.654566] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-aedb0b43-49ad-4ada-93dd-f3c585fe60f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.657171] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d7bf949-f66c-4e69-801a-9c7c0de0f786 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.670611] env[62506]: DEBUG nova.compute.provider_tree [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2055.676812] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2055.677698] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2055.677948] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fc1f024e-6830-4e9b-9ec0-9eae3e8c5853 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.683651] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2055.683651] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52f6e25b-5523-f024-89e7-2034bf3d2020" [ 2055.683651] env[62506]: _type = "Task" [ 2055.683651] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2055.691637] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52f6e25b-5523-f024-89e7-2034bf3d2020, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2055.699200] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f26c0701-71f1-4715-9cc4-e78df7c29325 tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "1970c9fb-368f-4fb5-80be-1b8601124bfe" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 105.676s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2055.941736] env[62506]: DEBUG nova.compute.manager [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Received event network-changed-d68a073f-0677-4771-bf93-5df437d2690a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2055.941943] env[62506]: DEBUG nova.compute.manager [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Refreshing instance network info cache due to event network-changed-d68a073f-0677-4771-bf93-5df437d2690a. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2055.942199] env[62506]: DEBUG oslo_concurrency.lockutils [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] Acquiring lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2055.942350] env[62506]: DEBUG oslo_concurrency.lockutils [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] Acquired lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2055.942511] env[62506]: DEBUG nova.network.neutron [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Refreshing network info cache for port d68a073f-0677-4771-bf93-5df437d2690a {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2056.047267] env[62506]: ERROR nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2056.047267] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2056.047267] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2056.047267] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2056.047267] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2056.047267] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2056.047267] env[62506]: ERROR nova.compute.manager raise self.value [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2056.047267] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2056.047267] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2056.047267] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2056.047720] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2056.047720] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2056.047720] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2056.047720] env[62506]: ERROR nova.compute.manager [ 2056.047720] env[62506]: Traceback (most recent call last): [ 2056.047720] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2056.047720] env[62506]: listener.cb(fileno) [ 2056.047720] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2056.047720] env[62506]: result = function(*args, **kwargs) [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2056.047720] env[62506]: return func(*args, **kwargs) [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2056.047720] env[62506]: raise e [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2056.047720] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2056.047720] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2056.047720] env[62506]: with excutils.save_and_reraise_exception(): [ 2056.047720] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2056.047720] env[62506]: self.force_reraise() [ 2056.047720] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2056.047720] env[62506]: raise self.value [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2056.047720] env[62506]: updated_port = self._update_port( [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2056.047720] env[62506]: _ensure_no_port_binding_failure(port) [ 2056.047720] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2056.047720] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2056.048491] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2056.048491] env[62506]: Removing descriptor: 20 [ 2056.179654] env[62506]: DEBUG nova.scheduler.client.report [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2056.203366] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2056.205939] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52f6e25b-5523-f024-89e7-2034bf3d2020, 'name': SearchDatastore_Task, 'duration_secs': 0.009127} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2056.208875] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-14d0ca41-a7f9-4db4-9c63-8c8da5bb7f82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.216491] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2056.216491] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5205f3b4-98b5-df23-510c-d2996c1effa6" [ 2056.216491] env[62506]: _type = "Task" [ 2056.216491] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2056.227341] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5205f3b4-98b5-df23-510c-d2996c1effa6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2056.356128] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2056.380932] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2056.381069] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2056.381281] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2056.381488] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2056.381635] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2056.381779] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2056.382014] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2056.382223] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2056.382397] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2056.382559] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2056.382730] env[62506]: DEBUG nova.virt.hardware [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2056.383623] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c6387c3-c5fc-46ed-929e-6fe2cb68c20f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.391437] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4848faa-f238-4f1d-8519-961aee04107f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.405284] env[62506]: ERROR nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Traceback (most recent call last): [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] yield resources [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self.driver.spawn(context, instance, image_meta, [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] vm_ref = self.build_virtual_machine(instance, [ 2056.405284] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] vif_infos = vmwarevif.get_vif_info(self._session, [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] for vif in network_info: [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] return self._sync_wrapper(fn, *args, **kwargs) [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self.wait() [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self[:] = self._gt.wait() [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] return self._exit_event.wait() [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2056.405597] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] current.throw(*self._exc) [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] result = function(*args, **kwargs) [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] return func(*args, **kwargs) [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] raise e [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] nwinfo = self.network_api.allocate_for_instance( [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] created_port_ids = self._update_ports_for_instance( [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] with excutils.save_and_reraise_exception(): [ 2056.405955] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self.force_reraise() [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] raise self.value [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] updated_port = self._update_port( [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] _ensure_no_port_binding_failure(port) [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] raise exception.PortBindingFailed(port_id=port['id']) [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2056.406288] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] [ 2056.406288] env[62506]: INFO nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Terminating instance [ 2056.461760] env[62506]: DEBUG nova.network.neutron [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2056.541352] env[62506]: DEBUG nova.network.neutron [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2056.684156] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.355s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2056.684697] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2056.690021] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.221s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2056.690021] env[62506]: INFO nova.compute.claims [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2056.727014] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5205f3b4-98b5-df23-510c-d2996c1effa6, 'name': SearchDatastore_Task, 'duration_secs': 0.010326} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2056.727300] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2056.727856] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2056.728168] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-24fa9f5b-10f9-4d89-bf2c-5733ceab779f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.731162] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2056.737513] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2056.737513] env[62506]: value = "task-2190853" [ 2056.737513] env[62506]: _type = "Task" [ 2056.737513] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2056.745579] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190853, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2056.910127] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2057.045383] env[62506]: DEBUG oslo_concurrency.lockutils [req-66d4b176-776f-4948-a35e-264ac6915d77 req-093058d2-c637-4c8c-9367-66797cefc889 service nova] Releasing lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2057.045829] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquired lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2057.046045] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2057.193948] env[62506]: DEBUG nova.compute.utils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2057.198051] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2057.198592] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2057.243287] env[62506]: DEBUG nova.policy [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '395a3d764baa47a094b40c75e34cc1b2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '003bcfc798e54853b5ac889661865a0b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2057.250323] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190853, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.451252} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2057.250598] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2057.251083] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2057.251083] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-b102d3d6-fc95-4d40-94c4-6ac4b0e22fe6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.257505] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2057.257505] env[62506]: value = "task-2190854" [ 2057.257505] env[62506]: _type = "Task" [ 2057.257505] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2057.265687] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190854, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2057.506757] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Successfully created port: 8717740a-13c8-40c4-b3e3-6764966e51d4 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2057.574600] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2057.699442] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2057.720265] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2057.771294] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190854, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.067715} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2057.771654] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2057.772797] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-501c73ca-b1d8-4e5c-b840-b319d5b0a043 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.796504] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Reconfiguring VM instance instance-00000034 to attach disk [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2057.796883] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-788ad3ba-8cb5-4e2b-96f5-5be719847a6a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.822741] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2057.822741] env[62506]: value = "task-2190855" [ 2057.822741] env[62506]: _type = "Task" [ 2057.822741] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2057.835429] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190855, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2057.976323] env[62506]: DEBUG nova.compute.manager [req-afab213a-121e-49d5-94af-2ebd079bdc0f req-5c623c4e-b9e4-41e9-be71-d8b7666cb25d service nova] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Received event network-vif-deleted-d68a073f-0677-4771-bf93-5df437d2690a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2058.014285] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64d83edb-497d-4287-9e83-0a0b0e11779c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.022187] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4631440-d692-4198-8d15-6938a68f2426 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.053546] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cccab94c-2ed1-4a2c-90cf-85385c3044e7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.061042] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eedc6d28-a77d-4957-a72f-17dc69046a4e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.075703] env[62506]: DEBUG nova.compute.provider_tree [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2058.223235] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Releasing lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2058.223650] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2058.223848] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2058.224153] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1b042012-ffde-4bc4-9b7a-cc083e61d839 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.233740] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8527d603-ffbd-4dbd-862e-f32d7c6c7502 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.259192] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 73ff1d78-5841-4532-a3f0-51e3f8e3314e could not be found. [ 2058.259477] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2058.259579] env[62506]: INFO nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2058.259819] env[62506]: DEBUG oslo.service.loopingcall [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2058.260072] env[62506]: DEBUG nova.compute.manager [-] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2058.260182] env[62506]: DEBUG nova.network.neutron [-] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2058.276590] env[62506]: DEBUG nova.network.neutron [-] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2058.333452] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190855, 'name': ReconfigVM_Task, 'duration_secs': 0.318046} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2058.333737] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Reconfigured VM instance instance-00000034 to attach disk [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828/8665bdf3-3930-4f45-b5be-e308f8f53828.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2058.334395] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-566992f5-f437-425a-b75b-a467452ba3bc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.341794] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2058.341794] env[62506]: value = "task-2190856" [ 2058.341794] env[62506]: _type = "Task" [ 2058.341794] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2058.347339] env[62506]: ERROR nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2058.347339] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2058.347339] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2058.347339] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2058.347339] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2058.347339] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2058.347339] env[62506]: ERROR nova.compute.manager raise self.value [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2058.347339] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2058.347339] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2058.347339] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2058.347745] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2058.347745] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2058.347745] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2058.347745] env[62506]: ERROR nova.compute.manager [ 2058.347745] env[62506]: Traceback (most recent call last): [ 2058.347745] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2058.347745] env[62506]: listener.cb(fileno) [ 2058.347745] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2058.347745] env[62506]: result = function(*args, **kwargs) [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2058.347745] env[62506]: return func(*args, **kwargs) [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2058.347745] env[62506]: raise e [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2058.347745] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2058.347745] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2058.347745] env[62506]: with excutils.save_and_reraise_exception(): [ 2058.347745] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2058.347745] env[62506]: self.force_reraise() [ 2058.347745] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2058.347745] env[62506]: raise self.value [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2058.347745] env[62506]: updated_port = self._update_port( [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2058.347745] env[62506]: _ensure_no_port_binding_failure(port) [ 2058.347745] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2058.347745] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2058.348407] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2058.348407] env[62506]: Removing descriptor: 20 [ 2058.354274] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190856, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2058.377370] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "80c76c85-2575-4733-a86a-111d138d69d4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2058.377611] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "80c76c85-2575-4733-a86a-111d138d69d4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2058.578476] env[62506]: DEBUG nova.scheduler.client.report [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2058.709825] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2058.736520] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2058.736520] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2058.736682] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2058.736826] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2058.736973] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2058.737136] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2058.737347] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2058.737511] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2058.737678] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2058.737837] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2058.738015] env[62506]: DEBUG nova.virt.hardware [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2058.738879] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4162eb90-bb5c-43b5-a0d3-8ab8adf941a3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.746591] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-238aab26-2605-43d5-a23d-d7836fe09052 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.760194] env[62506]: ERROR nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Traceback (most recent call last): [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] yield resources [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self.driver.spawn(context, instance, image_meta, [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] vm_ref = self.build_virtual_machine(instance, [ 2058.760194] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] vif_infos = vmwarevif.get_vif_info(self._session, [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] for vif in network_info: [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] return self._sync_wrapper(fn, *args, **kwargs) [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self.wait() [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self[:] = self._gt.wait() [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] return self._exit_event.wait() [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2058.760566] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] current.throw(*self._exc) [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] result = function(*args, **kwargs) [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] return func(*args, **kwargs) [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] raise e [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] nwinfo = self.network_api.allocate_for_instance( [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] created_port_ids = self._update_ports_for_instance( [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] with excutils.save_and_reraise_exception(): [ 2058.760922] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self.force_reraise() [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] raise self.value [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] updated_port = self._update_port( [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] _ensure_no_port_binding_failure(port) [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] raise exception.PortBindingFailed(port_id=port['id']) [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2058.761281] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] [ 2058.761281] env[62506]: INFO nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Terminating instance [ 2058.778660] env[62506]: DEBUG nova.network.neutron [-] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2058.850934] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190856, 'name': Rename_Task, 'duration_secs': 0.12805} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2058.851244] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2058.851481] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-66b946d3-3702-45d0-a186-c0bc0765927c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.859341] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2058.859341] env[62506]: value = "task-2190857" [ 2058.859341] env[62506]: _type = "Task" [ 2058.859341] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2058.868450] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190857, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2059.084185] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.397s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2059.084742] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2059.087875] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.819s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2059.089039] env[62506]: INFO nova.compute.claims [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2059.264978] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquiring lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2059.265366] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquired lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2059.265570] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2059.280779] env[62506]: INFO nova.compute.manager [-] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Took 1.02 seconds to deallocate network for instance. [ 2059.282930] env[62506]: DEBUG nova.compute.claims [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2059.283131] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2059.369893] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190857, 'name': PowerOnVM_Task} progress is 100%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2059.593428] env[62506]: DEBUG nova.compute.utils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2059.596816] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2059.596816] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2059.647519] env[62506]: DEBUG nova.policy [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f73527aacb5c43e7b78c5bdcfe24ed68', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'eb1f9277c5494d3f90e5c70248189824', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2059.786746] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2059.870108] env[62506]: DEBUG oslo_vmware.api [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190857, 'name': PowerOnVM_Task, 'duration_secs': 0.516781} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2059.870511] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2059.871192] env[62506]: DEBUG nova.compute.manager [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2059.871667] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2059.873281] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cb41e6b-63be-4716-b9ab-4b476764a10a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2059.934235] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Successfully created port: 870bc4bc-821e-4f63-97e0-726729147716 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2060.004757] env[62506]: DEBUG nova.compute.manager [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Received event network-changed-8717740a-13c8-40c4-b3e3-6764966e51d4 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2060.004961] env[62506]: DEBUG nova.compute.manager [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Refreshing instance network info cache due to event network-changed-8717740a-13c8-40c4-b3e3-6764966e51d4. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2060.005602] env[62506]: DEBUG oslo_concurrency.lockutils [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] Acquiring lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2060.098119] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2060.368314] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c67447c8-696c-41dc-ba45-b527c0f5f96b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2060.376154] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f7dd8a6-c392-4810-a8dc-e7d245ccffaf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2060.379597] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Releasing lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2060.380016] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2060.380229] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2060.380513] env[62506]: DEBUG oslo_concurrency.lockutils [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] Acquired lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2060.380684] env[62506]: DEBUG nova.network.neutron [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Refreshing network info cache for port 8717740a-13c8-40c4-b3e3-6764966e51d4 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2060.381660] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-da22eb8d-d1ca-4b04-a630-2dff03e2c05d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2060.414161] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64ee2b6d-3a9a-41e2-b1f4-70fb5a5e3dbb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2060.417861] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2060.421112] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb273fc4-e560-4a54-9d77-69e37ce5c747 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2060.435397] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fd2e4a6-266d-4b27-aeca-9e7c5c772943 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2060.446546] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7f33e930-8252-4019-81e1-79329a1ee1e3 could not be found. [ 2060.446772] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2060.446952] env[62506]: INFO nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Took 0.07 seconds to destroy the instance on the hypervisor. [ 2060.447219] env[62506]: DEBUG oslo.service.loopingcall [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2060.454866] env[62506]: DEBUG nova.compute.manager [-] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2060.454866] env[62506]: DEBUG nova.network.neutron [-] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2060.456983] env[62506]: DEBUG nova.compute.provider_tree [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2060.477774] env[62506]: DEBUG nova.network.neutron [-] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2060.790036] env[62506]: ERROR nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2060.790036] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2060.790036] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2060.790036] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2060.790036] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2060.790036] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2060.790036] env[62506]: ERROR nova.compute.manager raise self.value [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2060.790036] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2060.790036] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2060.790036] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2060.790728] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2060.790728] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2060.790728] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2060.790728] env[62506]: ERROR nova.compute.manager [ 2060.790728] env[62506]: Traceback (most recent call last): [ 2060.790728] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2060.790728] env[62506]: listener.cb(fileno) [ 2060.790728] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2060.790728] env[62506]: result = function(*args, **kwargs) [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2060.790728] env[62506]: return func(*args, **kwargs) [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2060.790728] env[62506]: raise e [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2060.790728] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2060.790728] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2060.790728] env[62506]: with excutils.save_and_reraise_exception(): [ 2060.790728] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2060.790728] env[62506]: self.force_reraise() [ 2060.790728] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2060.790728] env[62506]: raise self.value [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2060.790728] env[62506]: updated_port = self._update_port( [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2060.790728] env[62506]: _ensure_no_port_binding_failure(port) [ 2060.790728] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2060.790728] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2060.791504] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2060.791504] env[62506]: Removing descriptor: 20 [ 2060.899903] env[62506]: DEBUG nova.network.neutron [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2060.959866] env[62506]: DEBUG nova.scheduler.client.report [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2060.976703] env[62506]: DEBUG nova.network.neutron [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2060.981041] env[62506]: DEBUG nova.network.neutron [-] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2061.109270] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2061.135470] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2061.135748] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2061.135908] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2061.136078] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2061.136249] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2061.136396] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2061.136601] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2061.136759] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2061.136920] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2061.137092] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2061.137265] env[62506]: DEBUG nova.virt.hardware [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2061.138126] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f0dec71-6d02-48a4-b887-24f5ee7d898c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2061.145629] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-116acec5-0245-44f1-b1a6-8526502978d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2061.159137] env[62506]: ERROR nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Traceback (most recent call last): [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] yield resources [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self.driver.spawn(context, instance, image_meta, [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] vm_ref = self.build_virtual_machine(instance, [ 2061.159137] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] vif_infos = vmwarevif.get_vif_info(self._session, [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] for vif in network_info: [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] return self._sync_wrapper(fn, *args, **kwargs) [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self.wait() [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self[:] = self._gt.wait() [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] return self._exit_event.wait() [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2061.159506] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] current.throw(*self._exc) [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] result = function(*args, **kwargs) [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] return func(*args, **kwargs) [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] raise e [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] nwinfo = self.network_api.allocate_for_instance( [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] created_port_ids = self._update_ports_for_instance( [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] with excutils.save_and_reraise_exception(): [ 2061.159788] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self.force_reraise() [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] raise self.value [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] updated_port = self._update_port( [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] _ensure_no_port_binding_failure(port) [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] raise exception.PortBindingFailed(port_id=port['id']) [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2061.160189] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] [ 2061.160189] env[62506]: INFO nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Terminating instance [ 2061.315899] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "8665bdf3-3930-4f45-b5be-e308f8f53828" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2061.316182] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "8665bdf3-3930-4f45-b5be-e308f8f53828" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2061.316397] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "8665bdf3-3930-4f45-b5be-e308f8f53828-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2061.316581] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "8665bdf3-3930-4f45-b5be-e308f8f53828-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2061.316752] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "8665bdf3-3930-4f45-b5be-e308f8f53828-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2061.321803] env[62506]: INFO nova.compute.manager [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Terminating instance [ 2061.465259] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.378s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2061.465773] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2061.468295] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.623s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2061.479025] env[62506]: DEBUG oslo_concurrency.lockutils [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] Releasing lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2061.479175] env[62506]: DEBUG nova.compute.manager [req-1d53c519-7a35-47d2-9e7a-2386d9bd05d5 req-0610242c-654d-4b7d-bc09-ae9a27d08791 service nova] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Received event network-vif-deleted-8717740a-13c8-40c4-b3e3-6764966e51d4 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2061.484037] env[62506]: INFO nova.compute.manager [-] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Took 1.03 seconds to deallocate network for instance. [ 2061.485653] env[62506]: DEBUG nova.compute.claims [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2061.485776] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2061.668205] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2061.668352] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquired lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2061.668502] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2061.825082] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "refresh_cache-8665bdf3-3930-4f45-b5be-e308f8f53828" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2061.825265] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "refresh_cache-8665bdf3-3930-4f45-b5be-e308f8f53828" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2061.825396] env[62506]: DEBUG nova.network.neutron [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2061.973746] env[62506]: DEBUG nova.compute.utils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2061.979468] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2061.979468] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2062.034523] env[62506]: DEBUG nova.policy [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ab67b57f49ad47ea9aab58b4f7bc563f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c97e56471ed471aa4a0812ec92e082c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2062.047140] env[62506]: DEBUG nova.compute.manager [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Received event network-changed-870bc4bc-821e-4f63-97e0-726729147716 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2062.047368] env[62506]: DEBUG nova.compute.manager [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Refreshing instance network info cache due to event network-changed-870bc4bc-821e-4f63-97e0-726729147716. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2062.047528] env[62506]: DEBUG oslo_concurrency.lockutils [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] Acquiring lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2062.191139] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2062.232936] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d51f339-743d-49d0-bbd0-6653ec563fd6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.241455] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ee6504f-5e2e-497d-a8a0-ca9a15655ce4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.275627] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccfc713c-b49e-4cda-9dfe-765c7136cf8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.283711] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad7a44f3-e50c-4feb-94e4-5d1360347df2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.301433] env[62506]: DEBUG nova.compute.provider_tree [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2062.331841] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2062.343175] env[62506]: DEBUG nova.network.neutron [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2062.352753] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Successfully created port: 74764185-25a6-4229-9d2c-9858985d8956 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2062.390785] env[62506]: DEBUG nova.network.neutron [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2062.480161] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2062.804895] env[62506]: DEBUG nova.scheduler.client.report [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2062.835243] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Releasing lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2062.835658] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2062.835854] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2062.836397] env[62506]: DEBUG oslo_concurrency.lockutils [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] Acquired lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2062.836582] env[62506]: DEBUG nova.network.neutron [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Refreshing network info cache for port 870bc4bc-821e-4f63-97e0-726729147716 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2062.837637] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1024d29b-fde1-42c9-848e-5af6e07f06a2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.847351] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ba4e462-b712-4153-9117-4df5f5798894 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.871499] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b9edd77e-2f5a-4050-aaeb-c32b67d9aef7 could not be found. [ 2062.871726] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2062.871907] env[62506]: INFO nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2062.872186] env[62506]: DEBUG oslo.service.loopingcall [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2062.872408] env[62506]: DEBUG nova.compute.manager [-] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2062.872506] env[62506]: DEBUG nova.network.neutron [-] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2062.887793] env[62506]: DEBUG nova.network.neutron [-] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2062.895959] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "refresh_cache-8665bdf3-3930-4f45-b5be-e308f8f53828" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2062.896340] env[62506]: DEBUG nova.compute.manager [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2062.896527] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2062.897548] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bfb7abd-f4c9-4d27-9806-7fb104aa251b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.904888] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2062.905120] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-ebdb45bf-91ea-499d-8cbb-d03efacef98e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2062.910298] env[62506]: DEBUG oslo_vmware.api [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2062.910298] env[62506]: value = "task-2190858" [ 2062.910298] env[62506]: _type = "Task" [ 2062.910298] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2062.917687] env[62506]: DEBUG oslo_vmware.api [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190858, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2063.151209] env[62506]: ERROR nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2063.151209] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2063.151209] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2063.151209] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2063.151209] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2063.151209] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2063.151209] env[62506]: ERROR nova.compute.manager raise self.value [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2063.151209] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2063.151209] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2063.151209] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2063.151588] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2063.151588] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2063.151588] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2063.151588] env[62506]: ERROR nova.compute.manager [ 2063.151588] env[62506]: Traceback (most recent call last): [ 2063.151588] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2063.151588] env[62506]: listener.cb(fileno) [ 2063.151588] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2063.151588] env[62506]: result = function(*args, **kwargs) [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2063.151588] env[62506]: return func(*args, **kwargs) [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2063.151588] env[62506]: raise e [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2063.151588] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2063.151588] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2063.151588] env[62506]: with excutils.save_and_reraise_exception(): [ 2063.151588] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2063.151588] env[62506]: self.force_reraise() [ 2063.151588] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2063.151588] env[62506]: raise self.value [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2063.151588] env[62506]: updated_port = self._update_port( [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2063.151588] env[62506]: _ensure_no_port_binding_failure(port) [ 2063.151588] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2063.151588] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2063.152402] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2063.152402] env[62506]: Removing descriptor: 20 [ 2063.310201] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.842s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2063.310936] env[62506]: ERROR nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Traceback (most recent call last): [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self.driver.spawn(context, instance, image_meta, [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] vm_ref = self.build_virtual_machine(instance, [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] vif_infos = vmwarevif.get_vif_info(self._session, [ 2063.310936] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] for vif in network_info: [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] return self._sync_wrapper(fn, *args, **kwargs) [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self.wait() [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self[:] = self._gt.wait() [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] return self._exit_event.wait() [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] current.throw(*self._exc) [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2063.311231] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] result = function(*args, **kwargs) [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] return func(*args, **kwargs) [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] raise e [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] nwinfo = self.network_api.allocate_for_instance( [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] created_port_ids = self._update_ports_for_instance( [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] with excutils.save_and_reraise_exception(): [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] self.force_reraise() [ 2063.311596] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] raise self.value [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] updated_port = self._update_port( [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] _ensure_no_port_binding_failure(port) [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] raise exception.PortBindingFailed(port_id=port['id']) [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] nova.exception.PortBindingFailed: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. [ 2063.311877] env[62506]: ERROR nova.compute.manager [instance: dc0c0209-786b-44db-8f06-a429eec28bde] [ 2063.311877] env[62506]: DEBUG nova.compute.utils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2063.312970] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 13.200s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2063.313214] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2063.313402] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2063.313714] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.196s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2063.315191] env[62506]: INFO nova.compute.claims [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2063.317841] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Build of instance dc0c0209-786b-44db-8f06-a429eec28bde was re-scheduled: Binding failed for port 061c0690-c375-47ba-94a2-51828ae4c2a0, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2063.318280] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2063.318508] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquiring lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2063.318654] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Acquired lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2063.318813] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2063.320371] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ea44681-b553-4c1d-8164-0d4b66609471 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.329257] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42fd133c-5bac-44f0-8e55-f105ecb8ddcf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.345497] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48059b98-4853-4f0e-af04-5700b3d3ac61 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.353715] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24408adf-1f02-4dd8-8db6-2e2fb30bdc91 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.381669] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181279MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2063.381828] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2063.382731] env[62506]: DEBUG nova.network.neutron [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2063.389844] env[62506]: DEBUG nova.network.neutron [-] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2063.420980] env[62506]: DEBUG oslo_vmware.api [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190858, 'name': PowerOffVM_Task, 'duration_secs': 0.195156} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2063.421285] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2063.421398] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2063.421624] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0745da5b-83c1-4cbb-9852-5fa464fbe65e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.451810] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2063.452036] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2063.452299] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleting the datastore file [datastore2] 8665bdf3-3930-4f45-b5be-e308f8f53828 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2063.452550] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0f5fdf80-1254-46c0-980c-7e274d9848d6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.458787] env[62506]: DEBUG nova.network.neutron [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2063.460666] env[62506]: DEBUG oslo_vmware.api [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2063.460666] env[62506]: value = "task-2190860" [ 2063.460666] env[62506]: _type = "Task" [ 2063.460666] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2063.468606] env[62506]: DEBUG oslo_vmware.api [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190860, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2063.488958] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2063.512856] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2063.513141] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2063.513331] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2063.513535] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2063.513703] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2063.513869] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2063.514103] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2063.514276] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2063.514461] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2063.514637] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2063.514824] env[62506]: DEBUG nova.virt.hardware [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2063.515707] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a08cb0ad-a9bf-4b2d-a849-b7bc933b270b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.523524] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8161668d-b37a-4a39-86ce-d0c6380911bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.537709] env[62506]: ERROR nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Traceback (most recent call last): [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] yield resources [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self.driver.spawn(context, instance, image_meta, [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] vm_ref = self.build_virtual_machine(instance, [ 2063.537709] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] vif_infos = vmwarevif.get_vif_info(self._session, [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] for vif in network_info: [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] return self._sync_wrapper(fn, *args, **kwargs) [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self.wait() [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self[:] = self._gt.wait() [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] return self._exit_event.wait() [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2063.538249] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] current.throw(*self._exc) [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] result = function(*args, **kwargs) [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] return func(*args, **kwargs) [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] raise e [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] nwinfo = self.network_api.allocate_for_instance( [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] created_port_ids = self._update_ports_for_instance( [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] with excutils.save_and_reraise_exception(): [ 2063.538749] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self.force_reraise() [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] raise self.value [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] updated_port = self._update_port( [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] _ensure_no_port_binding_failure(port) [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] raise exception.PortBindingFailed(port_id=port['id']) [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2063.539270] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] [ 2063.539270] env[62506]: INFO nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Terminating instance [ 2063.842349] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2063.892315] env[62506]: INFO nova.compute.manager [-] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Took 1.02 seconds to deallocate network for instance. [ 2063.894489] env[62506]: DEBUG nova.compute.claims [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2063.894672] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2063.919815] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2063.962169] env[62506]: DEBUG oslo_concurrency.lockutils [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] Releasing lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2063.962473] env[62506]: DEBUG nova.compute.manager [req-78c226ba-36f9-42de-9736-e6cacab7304b req-cb714f5c-1ca6-40ef-859b-e52062617613 service nova] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Received event network-vif-deleted-870bc4bc-821e-4f63-97e0-726729147716 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2063.972071] env[62506]: DEBUG oslo_vmware.api [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190860, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.083196} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2063.972355] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2063.972591] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2063.972787] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2063.972957] env[62506]: INFO nova.compute.manager [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Took 1.08 seconds to destroy the instance on the hypervisor. [ 2063.973228] env[62506]: DEBUG oslo.service.loopingcall [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2063.973430] env[62506]: DEBUG nova.compute.manager [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2063.973526] env[62506]: DEBUG nova.network.neutron [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2063.990135] env[62506]: DEBUG nova.network.neutron [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2064.041680] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2064.041867] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2064.042147] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2064.072333] env[62506]: DEBUG nova.compute.manager [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Received event network-changed-74764185-25a6-4229-9d2c-9858985d8956 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2064.072581] env[62506]: DEBUG nova.compute.manager [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Refreshing instance network info cache due to event network-changed-74764185-25a6-4229-9d2c-9858985d8956. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2064.072774] env[62506]: DEBUG oslo_concurrency.lockutils [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] Acquiring lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2064.423730] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Releasing lock "refresh_cache-dc0c0209-786b-44db-8f06-a429eec28bde" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2064.423967] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2064.424177] env[62506]: DEBUG nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2064.424349] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2064.440350] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2064.493256] env[62506]: DEBUG nova.network.neutron [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2064.538687] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b0f517b-7fc8-4c1c-bdba-19a4b52fd65b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2064.547922] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a245c8ac-f8da-49a5-aed5-5c7ce71b43a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2064.579984] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2064.582210] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f669788a-75e9-4708-93d3-f22039de6d25 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2064.590078] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee855dfd-0ddc-4383-b5bf-9fcc759cc17f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2064.603445] env[62506]: DEBUG nova.compute.provider_tree [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2064.652950] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2064.943646] env[62506]: DEBUG nova.network.neutron [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2064.995508] env[62506]: INFO nova.compute.manager [-] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Took 1.02 seconds to deallocate network for instance. [ 2065.106032] env[62506]: DEBUG nova.scheduler.client.report [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2065.155599] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2065.156068] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2065.156279] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2065.156911] env[62506]: DEBUG oslo_concurrency.lockutils [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] Acquired lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2065.157104] env[62506]: DEBUG nova.network.neutron [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Refreshing network info cache for port 74764185-25a6-4229-9d2c-9858985d8956 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2065.158144] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b8085822-efd2-4b11-ab64-31044adcfc00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2065.168199] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29530edf-c6bb-4d6c-8c8b-90abf56ee17f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2065.193965] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 56c46b60-5feb-4b7a-8c4c-b010f4cf7009 could not be found. [ 2065.194332] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2065.194527] env[62506]: INFO nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2065.194762] env[62506]: DEBUG oslo.service.loopingcall [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2065.194982] env[62506]: DEBUG nova.compute.manager [-] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2065.195091] env[62506]: DEBUG nova.network.neutron [-] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2065.210031] env[62506]: DEBUG nova.network.neutron [-] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2065.446298] env[62506]: INFO nova.compute.manager [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] [instance: dc0c0209-786b-44db-8f06-a429eec28bde] Took 1.02 seconds to deallocate network for instance. [ 2065.501902] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2065.610969] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.297s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2065.611555] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2065.614857] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.855s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2065.616249] env[62506]: INFO nova.compute.claims [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2065.676118] env[62506]: DEBUG nova.network.neutron [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2065.712029] env[62506]: DEBUG nova.network.neutron [-] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2065.748768] env[62506]: DEBUG nova.network.neutron [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2066.120213] env[62506]: DEBUG nova.compute.utils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2066.123387] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2066.123594] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2066.167553] env[62506]: DEBUG nova.policy [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '69c10cca15644239bf7b17e65d48c658', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '906a5d81ca404211b5a28f8683248ef1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2066.216510] env[62506]: INFO nova.compute.manager [-] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Took 1.02 seconds to deallocate network for instance. [ 2066.218895] env[62506]: DEBUG nova.compute.claims [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2066.219331] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2066.250897] env[62506]: DEBUG oslo_concurrency.lockutils [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] Releasing lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2066.251184] env[62506]: DEBUG nova.compute.manager [req-fa3f3784-ff5e-4946-a2ea-2da2cde4abec req-ece3ae67-940b-4133-aa08-16ef51e1c358 service nova] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Received event network-vif-deleted-74764185-25a6-4229-9d2c-9858985d8956 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2066.451948] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Successfully created port: a57b4361-4fee-4c7d-b32a-75abd59ec9ca {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2066.476324] env[62506]: INFO nova.scheduler.client.report [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Deleted allocations for instance dc0c0209-786b-44db-8f06-a429eec28bde [ 2066.626767] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2066.854860] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcb06518-18b7-4e74-95c5-5a08ce73dcaa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2066.862492] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99fa6f12-2354-4f9a-96ea-752a92f68021 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2066.894398] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1841b7d-6017-44e6-8a1b-db74f31a30d7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2066.902407] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6be4fbfe-ca0b-4ba2-80ed-bbb823910748 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2066.915706] env[62506]: DEBUG nova.compute.provider_tree [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2066.985173] env[62506]: DEBUG oslo_concurrency.lockutils [None req-feba97d4-e83c-4039-84ae-02a522731f8e tempest-ServerActionsV293TestJSON-1778252471 tempest-ServerActionsV293TestJSON-1778252471-project-member] Lock "dc0c0209-786b-44db-8f06-a429eec28bde" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 107.103s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2067.132157] env[62506]: DEBUG nova.compute.manager [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Received event network-changed-a57b4361-4fee-4c7d-b32a-75abd59ec9ca {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2067.132157] env[62506]: DEBUG nova.compute.manager [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Refreshing instance network info cache due to event network-changed-a57b4361-4fee-4c7d-b32a-75abd59ec9ca. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2067.132157] env[62506]: DEBUG oslo_concurrency.lockutils [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] Acquiring lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2067.132157] env[62506]: DEBUG oslo_concurrency.lockutils [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] Acquired lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2067.132157] env[62506]: DEBUG nova.network.neutron [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Refreshing network info cache for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2067.419317] env[62506]: DEBUG nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2067.423395] env[62506]: ERROR nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2067.423395] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2067.423395] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2067.423395] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2067.423395] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2067.423395] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2067.423395] env[62506]: ERROR nova.compute.manager raise self.value [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2067.423395] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2067.423395] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2067.423395] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2067.423837] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2067.423837] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2067.423837] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2067.423837] env[62506]: ERROR nova.compute.manager [ 2067.423837] env[62506]: Traceback (most recent call last): [ 2067.423837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2067.423837] env[62506]: listener.cb(fileno) [ 2067.423837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2067.423837] env[62506]: result = function(*args, **kwargs) [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2067.423837] env[62506]: return func(*args, **kwargs) [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2067.423837] env[62506]: raise e [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2067.423837] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2067.423837] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2067.423837] env[62506]: with excutils.save_and_reraise_exception(): [ 2067.423837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2067.423837] env[62506]: self.force_reraise() [ 2067.423837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2067.423837] env[62506]: raise self.value [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2067.423837] env[62506]: updated_port = self._update_port( [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2067.423837] env[62506]: _ensure_no_port_binding_failure(port) [ 2067.423837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2067.423837] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2067.424702] env[62506]: nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2067.424702] env[62506]: Removing descriptor: 20 [ 2067.487433] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2067.641678] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2067.652010] env[62506]: DEBUG nova.network.neutron [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2067.669459] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2067.670216] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2067.670216] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2067.670216] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2067.670434] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2067.670479] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2067.670855] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2067.670913] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2067.671093] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2067.671267] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2067.671439] env[62506]: DEBUG nova.virt.hardware [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2067.672433] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea56fea7-bef3-4191-9cd6-894dbebeeb11 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2067.680580] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2384137b-1767-42f6-8c49-cba6ce93f94f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2067.695478] env[62506]: ERROR nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Traceback (most recent call last): [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] yield resources [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self.driver.spawn(context, instance, image_meta, [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] vm_ref = self.build_virtual_machine(instance, [ 2067.695478] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] vif_infos = vmwarevif.get_vif_info(self._session, [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] for vif in network_info: [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] return self._sync_wrapper(fn, *args, **kwargs) [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self.wait() [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self[:] = self._gt.wait() [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] return self._exit_event.wait() [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2067.695980] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] current.throw(*self._exc) [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] result = function(*args, **kwargs) [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] return func(*args, **kwargs) [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] raise e [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] nwinfo = self.network_api.allocate_for_instance( [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] created_port_ids = self._update_ports_for_instance( [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] with excutils.save_and_reraise_exception(): [ 2067.696423] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self.force_reraise() [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] raise self.value [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] updated_port = self._update_port( [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] _ensure_no_port_binding_failure(port) [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] raise exception.PortBindingFailed(port_id=port['id']) [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2067.696782] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] [ 2067.696782] env[62506]: INFO nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Terminating instance [ 2067.741027] env[62506]: DEBUG nova.network.neutron [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2067.925210] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.310s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2067.925769] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2067.928340] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.255s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2068.016227] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2068.202518] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2068.243792] env[62506]: DEBUG oslo_concurrency.lockutils [req-bd7a8d57-3889-47af-b7ea-95f06d982057 req-b5a36c03-00e1-4b64-8335-b656a53938d3 service nova] Releasing lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2068.244186] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquired lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2068.244406] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2068.432601] env[62506]: DEBUG nova.compute.utils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2068.436625] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2068.436806] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2068.495148] env[62506]: DEBUG nova.policy [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '69c10cca15644239bf7b17e65d48c658', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '906a5d81ca404211b5a28f8683248ef1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2068.673835] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7f4a2e1-c09b-4fc9-adff-fc7dfc479c6c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.681838] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-535165eb-ee7c-42a7-ac91-d4416c3ffc8a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.711408] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52cdb0c7-6e9c-47dc-84d8-ba934c60e562 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.718827] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc4d1323-e9b5-4046-a869-9b893fe9f176 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.731834] env[62506]: DEBUG nova.compute.provider_tree [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2068.767214] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2068.856820] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2068.911632] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Successfully created port: 6b70fa38-c05c-476f-8129-d1adeca8a94d {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2068.939794] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2069.165875] env[62506]: DEBUG nova.compute.manager [req-63c1c208-9d66-4da7-b395-fbe965d3077b req-e6f7abf6-0920-48c6-af52-418b6fad3d55 service nova] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Received event network-vif-deleted-a57b4361-4fee-4c7d-b32a-75abd59ec9ca {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2069.238118] env[62506]: DEBUG nova.scheduler.client.report [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2069.363096] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Releasing lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2069.363096] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2069.363096] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2069.363096] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0d49feea-aae8-466f-ab5b-2add76455e88 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.371742] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd435bfe-eb85-49ef-856d-38b5f3e8c09a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.400241] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 635dfc2e-d53a-467a-a23e-ee927ff5b2b1 could not be found. [ 2069.400241] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2069.400241] env[62506]: INFO nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2069.400241] env[62506]: DEBUG oslo.service.loopingcall [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2069.400241] env[62506]: DEBUG nova.compute.manager [-] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2069.400241] env[62506]: DEBUG nova.network.neutron [-] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2069.420208] env[62506]: DEBUG nova.network.neutron [-] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2069.746211] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.816s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2069.746211] env[62506]: ERROR nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Traceback (most recent call last): [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self.driver.spawn(context, instance, image_meta, [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2069.746211] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] vm_ref = self.build_virtual_machine(instance, [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] vif_infos = vmwarevif.get_vif_info(self._session, [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] for vif in network_info: [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] return self._sync_wrapper(fn, *args, **kwargs) [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self.wait() [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self[:] = self._gt.wait() [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] return self._exit_event.wait() [ 2069.746671] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] current.throw(*self._exc) [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] result = function(*args, **kwargs) [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] return func(*args, **kwargs) [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] raise e [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] nwinfo = self.network_api.allocate_for_instance( [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] created_port_ids = self._update_ports_for_instance( [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2069.746952] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] with excutils.save_and_reraise_exception(): [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] self.force_reraise() [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] raise self.value [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] updated_port = self._update_port( [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] _ensure_no_port_binding_failure(port) [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] raise exception.PortBindingFailed(port_id=port['id']) [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] nova.exception.PortBindingFailed: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. [ 2069.747278] env[62506]: ERROR nova.compute.manager [instance: 97091b4f-8448-49e5-9382-99e3358300c6] [ 2069.747604] env[62506]: DEBUG nova.compute.utils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2069.747742] env[62506]: ERROR nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2069.747742] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2069.747742] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2069.747742] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2069.747742] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2069.747742] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2069.747742] env[62506]: ERROR nova.compute.manager raise self.value [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2069.747742] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2069.747742] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2069.747742] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2069.748309] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2069.748309] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2069.748309] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2069.748309] env[62506]: ERROR nova.compute.manager [ 2069.748417] env[62506]: Traceback (most recent call last): [ 2069.748447] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2069.748447] env[62506]: listener.cb(fileno) [ 2069.748447] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2069.748447] env[62506]: result = function(*args, **kwargs) [ 2069.748447] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2069.748447] env[62506]: return func(*args, **kwargs) [ 2069.748447] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2069.748447] env[62506]: raise e [ 2069.748447] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2069.748447] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2069.748447] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2069.748447] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2069.748447] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2069.748447] env[62506]: with excutils.save_and_reraise_exception(): [ 2069.748447] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2069.748447] env[62506]: self.force_reraise() [ 2069.748447] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2069.748447] env[62506]: raise self.value [ 2069.748905] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2069.748905] env[62506]: updated_port = self._update_port( [ 2069.748905] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2069.748905] env[62506]: _ensure_no_port_binding_failure(port) [ 2069.748905] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2069.748905] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2069.748905] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2069.748905] env[62506]: Removing descriptor: 20 [ 2069.749195] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.837s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2069.752956] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Build of instance 97091b4f-8448-49e5-9382-99e3358300c6 was re-scheduled: Binding failed for port 7affea15-4208-4851-91bd-9432223afd70, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2069.753391] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2069.753626] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2069.753776] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2069.753935] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2069.922709] env[62506]: DEBUG nova.network.neutron [-] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2069.949789] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2069.975182] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2069.975435] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2069.975592] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2069.975775] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2069.975922] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2069.976080] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2069.976292] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2069.976454] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2069.976619] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2069.976779] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2069.976957] env[62506]: DEBUG nova.virt.hardware [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2069.977814] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1a7d57d-9fe3-44fb-89ed-9750b9f3e4bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.985728] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1726ce0f-748b-4b43-b6a6-bc96de404da2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.999397] env[62506]: ERROR nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] Traceback (most recent call last): [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] yield resources [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self.driver.spawn(context, instance, image_meta, [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] vm_ref = self.build_virtual_machine(instance, [ 2069.999397] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] for vif in network_info: [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] return self._sync_wrapper(fn, *args, **kwargs) [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self.wait() [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self[:] = self._gt.wait() [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] return self._exit_event.wait() [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2069.999772] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] current.throw(*self._exc) [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] result = function(*args, **kwargs) [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] return func(*args, **kwargs) [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] raise e [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] nwinfo = self.network_api.allocate_for_instance( [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] created_port_ids = self._update_ports_for_instance( [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] with excutils.save_and_reraise_exception(): [ 2070.000213] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self.force_reraise() [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] raise self.value [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] updated_port = self._update_port( [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] _ensure_no_port_binding_failure(port) [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] raise exception.PortBindingFailed(port_id=port['id']) [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2070.000551] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] [ 2070.000551] env[62506]: INFO nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Terminating instance [ 2070.274948] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2070.360084] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2070.424977] env[62506]: INFO nova.compute.manager [-] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Took 1.03 seconds to deallocate network for instance. [ 2070.429019] env[62506]: DEBUG nova.compute.claims [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2070.429019] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2070.460733] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d912f1b-e574-4ccb-bde3-a7b1cdee5c62 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.468687] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97aa9927-8c58-4e85-ae29-05786826d32b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.497905] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54c268fe-70fb-46a1-9632-1cf8bf603b26 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.504786] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2070.504961] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquired lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2070.505151] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2070.507156] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58c6b1cb-e62b-4689-82ac-1150955348da {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2070.520609] env[62506]: DEBUG nova.compute.provider_tree [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2070.862574] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-97091b4f-8448-49e5-9382-99e3358300c6" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2070.862925] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2070.862972] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2070.863163] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2070.879907] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2071.023746] env[62506]: DEBUG nova.scheduler.client.report [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2071.028059] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2071.102071] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2071.189301] env[62506]: DEBUG nova.compute.manager [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Received event network-changed-6b70fa38-c05c-476f-8129-d1adeca8a94d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2071.189511] env[62506]: DEBUG nova.compute.manager [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Refreshing instance network info cache due to event network-changed-6b70fa38-c05c-476f-8129-d1adeca8a94d. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2071.189706] env[62506]: DEBUG oslo_concurrency.lockutils [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] Acquiring lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2071.382292] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2071.531017] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.782s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2071.531486] env[62506]: ERROR nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Traceback (most recent call last): [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self.driver.spawn(context, instance, image_meta, [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] vm_ref = self.build_virtual_machine(instance, [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] vif_infos = vmwarevif.get_vif_info(self._session, [ 2071.531486] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] for vif in network_info: [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] return self._sync_wrapper(fn, *args, **kwargs) [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self.wait() [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self[:] = self._gt.wait() [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] return self._exit_event.wait() [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] current.throw(*self._exc) [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2071.531883] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] result = function(*args, **kwargs) [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] return func(*args, **kwargs) [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] raise e [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] nwinfo = self.network_api.allocate_for_instance( [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] created_port_ids = self._update_ports_for_instance( [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] with excutils.save_and_reraise_exception(): [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] self.force_reraise() [ 2071.532229] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] raise self.value [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] updated_port = self._update_port( [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] _ensure_no_port_binding_failure(port) [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] raise exception.PortBindingFailed(port_id=port['id']) [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] nova.exception.PortBindingFailed: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. [ 2071.532514] env[62506]: ERROR nova.compute.manager [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] [ 2071.532514] env[62506]: DEBUG nova.compute.utils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2071.533394] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.802s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2071.534837] env[62506]: INFO nova.compute.claims [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2071.537517] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Build of instance 50b8f972-e2bd-4526-ab28-bf887a240421 was re-scheduled: Binding failed for port 624ec51d-a602-4a9a-aae1-0e1738502bf6, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2071.537932] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2071.538166] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2071.538314] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2071.538470] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2071.604557] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Releasing lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2071.606751] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2071.606751] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2071.606751] env[62506]: DEBUG oslo_concurrency.lockutils [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] Acquired lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2071.606751] env[62506]: DEBUG nova.network.neutron [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Refreshing network info cache for port 6b70fa38-c05c-476f-8129-d1adeca8a94d {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2071.606751] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-39384dc3-21c0-4353-b6eb-96e8939d8b44 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2071.617028] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bbef366-a3bf-46b9-a3f4-e930b32f8ae1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2071.639571] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 11614111-9445-4299-8626-d65d2038b0cd could not be found. [ 2071.639804] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2071.639987] env[62506]: INFO nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2071.640277] env[62506]: DEBUG oslo.service.loopingcall [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2071.640754] env[62506]: DEBUG nova.compute.manager [-] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2071.640851] env[62506]: DEBUG nova.network.neutron [-] [instance: 11614111-9445-4299-8626-d65d2038b0cd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2071.656439] env[62506]: DEBUG nova.network.neutron [-] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2071.884770] env[62506]: INFO nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 97091b4f-8448-49e5-9382-99e3358300c6] Took 1.02 seconds to deallocate network for instance. [ 2072.056352] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2072.123424] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2072.125573] env[62506]: DEBUG nova.network.neutron [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2072.158609] env[62506]: DEBUG nova.network.neutron [-] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2072.227268] env[62506]: DEBUG nova.network.neutron [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2072.626045] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-50b8f972-e2bd-4526-ab28-bf887a240421" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2072.626045] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2072.626158] env[62506]: DEBUG nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2072.626276] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2072.649203] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2072.660833] env[62506]: INFO nova.compute.manager [-] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Took 1.02 seconds to deallocate network for instance. [ 2072.663123] env[62506]: DEBUG nova.compute.claims [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2072.663236] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2072.729367] env[62506]: DEBUG oslo_concurrency.lockutils [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] Releasing lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2072.729630] env[62506]: DEBUG nova.compute.manager [req-c775095a-a07c-4abd-bfaf-0061d92c7b32 req-93d2997a-c367-4844-8702-ce1e51a0ef1a service nova] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Received event network-vif-deleted-6b70fa38-c05c-476f-8129-d1adeca8a94d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2072.773075] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-679a1e38-7ef1-4e21-b8cb-c70bc2bf1a31 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2072.782307] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-921e2d0a-4a26-4ae7-aec2-0f73afdf0ebf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2072.812768] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6fbcf10-0c62-4043-8ae4-a506e9a00ada {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2072.820445] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d79e006-053d-4c42-9793-c91f26593027 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2072.833769] env[62506]: DEBUG nova.compute.provider_tree [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2072.913440] env[62506]: INFO nova.scheduler.client.report [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Deleted allocations for instance 97091b4f-8448-49e5-9382-99e3358300c6 [ 2073.153041] env[62506]: DEBUG nova.network.neutron [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2073.336681] env[62506]: DEBUG nova.scheduler.client.report [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2073.424419] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "97091b4f-8448-49e5-9382-99e3358300c6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 106.920s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2073.655845] env[62506]: INFO nova.compute.manager [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 50b8f972-e2bd-4526-ab28-bf887a240421] Took 1.03 seconds to deallocate network for instance. [ 2073.845047] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.311s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2073.845537] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2073.848115] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.565s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2073.926639] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2074.355953] env[62506]: DEBUG nova.compute.utils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2074.357851] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2074.357851] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2074.403665] env[62506]: DEBUG nova.policy [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '520fe05ea52d4eea98ca08e4b1d73811', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '959b8b2dd1e047af96e3705eff02d631', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2074.444858] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2074.546740] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32b3593a-c2e6-42af-abbf-e270e6f8791e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2074.554987] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-399d53a6-6cd7-4124-a605-1be15249cbe9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2074.584399] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ddb5b4b-cd8c-485f-ab51-620b5f4a513c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2074.591477] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2d42455-b0b0-45c7-ad8e-8336b6b108c5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2074.604093] env[62506]: DEBUG nova.compute.provider_tree [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2074.686216] env[62506]: INFO nova.scheduler.client.report [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Deleted allocations for instance 50b8f972-e2bd-4526-ab28-bf887a240421 [ 2074.695332] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Successfully created port: b1c6ec19-23aa-47c5-bcba-b80f29b679a7 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2074.860642] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2075.107786] env[62506]: DEBUG nova.scheduler.client.report [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2075.199852] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2cf7795f-34ea-4330-b067-a12ddd8e4246 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "50b8f972-e2bd-4526-ab28-bf887a240421" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 108.672s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2075.496774] env[62506]: DEBUG nova.compute.manager [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Received event network-changed-b1c6ec19-23aa-47c5-bcba-b80f29b679a7 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2075.496948] env[62506]: DEBUG nova.compute.manager [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Refreshing instance network info cache due to event network-changed-b1c6ec19-23aa-47c5-bcba-b80f29b679a7. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2075.497188] env[62506]: DEBUG oslo_concurrency.lockutils [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] Acquiring lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2075.497338] env[62506]: DEBUG oslo_concurrency.lockutils [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] Acquired lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2075.497500] env[62506]: DEBUG nova.network.neutron [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Refreshing network info cache for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2075.612493] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.764s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2075.613318] env[62506]: ERROR nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Traceback (most recent call last): [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self.driver.spawn(context, instance, image_meta, [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] vm_ref = self.build_virtual_machine(instance, [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] vif_infos = vmwarevif.get_vif_info(self._session, [ 2075.613318] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] for vif in network_info: [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] return self._sync_wrapper(fn, *args, **kwargs) [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self.wait() [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self[:] = self._gt.wait() [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] return self._exit_event.wait() [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] current.throw(*self._exc) [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2075.613683] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] result = function(*args, **kwargs) [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] return func(*args, **kwargs) [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] raise e [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] nwinfo = self.network_api.allocate_for_instance( [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] created_port_ids = self._update_ports_for_instance( [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] with excutils.save_and_reraise_exception(): [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] self.force_reraise() [ 2075.614090] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] raise self.value [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] updated_port = self._update_port( [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] _ensure_no_port_binding_failure(port) [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] raise exception.PortBindingFailed(port_id=port['id']) [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] nova.exception.PortBindingFailed: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. [ 2075.614482] env[62506]: ERROR nova.compute.manager [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] [ 2075.614482] env[62506]: DEBUG nova.compute.utils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2075.617883] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 15.200s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2075.617883] env[62506]: DEBUG nova.objects.instance [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 8665bdf3-3930-4f45-b5be-e308f8f53828] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62506) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 2075.620868] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Build of instance 73ff1d78-5841-4532-a3f0-51e3f8e3314e was re-scheduled: Binding failed for port d68a073f-0677-4771-bf93-5df437d2690a, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2075.621049] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2075.623701] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquiring lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2075.623701] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Acquired lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2075.623701] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2075.658399] env[62506]: ERROR nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2075.658399] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2075.658399] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2075.658399] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2075.658399] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2075.658399] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2075.658399] env[62506]: ERROR nova.compute.manager raise self.value [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2075.658399] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2075.658399] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2075.658399] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2075.658894] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2075.658894] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2075.658894] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2075.658894] env[62506]: ERROR nova.compute.manager [ 2075.658894] env[62506]: Traceback (most recent call last): [ 2075.658894] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2075.658894] env[62506]: listener.cb(fileno) [ 2075.658894] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2075.658894] env[62506]: result = function(*args, **kwargs) [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2075.658894] env[62506]: return func(*args, **kwargs) [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2075.658894] env[62506]: raise e [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2075.658894] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2075.658894] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2075.658894] env[62506]: with excutils.save_and_reraise_exception(): [ 2075.658894] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2075.658894] env[62506]: self.force_reraise() [ 2075.658894] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2075.658894] env[62506]: raise self.value [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2075.658894] env[62506]: updated_port = self._update_port( [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2075.658894] env[62506]: _ensure_no_port_binding_failure(port) [ 2075.658894] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2075.658894] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2075.659684] env[62506]: nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2075.659684] env[62506]: Removing descriptor: 19 [ 2075.702294] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2075.870030] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2075.889122] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T14:03:19Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='ad8cfa99-3724-4f34-8dfc-f9febd18ac6b',id=38,is_public=True,memory_mb=192,name='tempest-test_resize_flavor_-1293260714',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2075.889378] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2075.889690] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2075.889813] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2075.889916] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2075.890079] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2075.890395] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2075.890563] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2075.890739] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2075.890894] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2075.891078] env[62506]: DEBUG nova.virt.hardware [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2075.891984] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-528fddc2-9b49-45ba-a48a-108419af18ee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2075.900281] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39ade5f9-3973-4373-baa5-98cc7587afc9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2075.913991] env[62506]: ERROR nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Traceback (most recent call last): [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] yield resources [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self.driver.spawn(context, instance, image_meta, [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] vm_ref = self.build_virtual_machine(instance, [ 2075.913991] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] vif_infos = vmwarevif.get_vif_info(self._session, [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] for vif in network_info: [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] return self._sync_wrapper(fn, *args, **kwargs) [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self.wait() [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self[:] = self._gt.wait() [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] return self._exit_event.wait() [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2075.914505] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] current.throw(*self._exc) [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] result = function(*args, **kwargs) [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] return func(*args, **kwargs) [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] raise e [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] nwinfo = self.network_api.allocate_for_instance( [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] created_port_ids = self._update_ports_for_instance( [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] with excutils.save_and_reraise_exception(): [ 2075.914846] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self.force_reraise() [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] raise self.value [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] updated_port = self._update_port( [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] _ensure_no_port_binding_failure(port) [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] raise exception.PortBindingFailed(port_id=port['id']) [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2075.915194] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] [ 2075.915194] env[62506]: INFO nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Terminating instance [ 2076.024099] env[62506]: DEBUG nova.network.neutron [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2076.115699] env[62506]: DEBUG nova.network.neutron [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2076.143948] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2076.216737] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2076.226133] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2076.417849] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2076.618976] env[62506]: DEBUG oslo_concurrency.lockutils [req-9c2a7f9b-04a6-4ad0-8640-0bb305fff0f5 req-6ac332af-e31c-43fd-ba31-6b637feee10f service nova] Releasing lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2076.619416] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquired lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2076.619618] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2076.629777] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7be2f6f3-78f4-47fd-915f-8c0b46641a5d tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.012s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2076.631059] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.145s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2076.719248] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Releasing lock "refresh_cache-73ff1d78-5841-4532-a3f0-51e3f8e3314e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2076.719449] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2076.719710] env[62506]: DEBUG nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2076.719918] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2076.736394] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2076.875924] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "1147abf9-da6d-4e51-878e-b8dc9a3f6141" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2076.876190] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "1147abf9-da6d-4e51-878e-b8dc9a3f6141" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2076.904536] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "d4e49d27-403b-4f85-95b7-ae07e9bb07b8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2076.904847] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "d4e49d27-403b-4f85-95b7-ae07e9bb07b8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2077.145998] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2077.206610] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2077.238469] env[62506]: DEBUG nova.network.neutron [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2077.349576] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c28851fd-8709-4387-a1a6-f54da2763f22 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.357936] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6efa8bb0-ac8b-4ccb-adfb-fc38a57bd273 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.389981] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51bae53f-6a1c-42db-bcaa-ffbfa71bd22c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.397482] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8beea01f-e9de-4370-9781-e29b42eb8016 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.411198] env[62506]: DEBUG nova.compute.provider_tree [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2077.522725] env[62506]: DEBUG nova.compute.manager [req-6da1b1a2-9ce4-44ff-abff-fead6ad11dd9 req-d1770c5c-1e87-4c31-a113-7e5cb4c9ddb1 service nova] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Received event network-vif-deleted-b1c6ec19-23aa-47c5-bcba-b80f29b679a7 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2077.709388] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Releasing lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2077.710028] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2077.710331] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2077.711031] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4a72fe8c-e6f0-4c33-bc46-5acd7a93a880 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.720210] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-906598e8-5611-43e0-8f00-910360c45666 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.743507] env[62506]: INFO nova.compute.manager [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] [instance: 73ff1d78-5841-4532-a3f0-51e3f8e3314e] Took 1.02 seconds to deallocate network for instance. [ 2077.747354] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bd441dae-ac8b-41bd-8fae-15dca58f820e could not be found. [ 2077.747593] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2077.747783] env[62506]: INFO nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2077.748098] env[62506]: DEBUG oslo.service.loopingcall [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2077.748573] env[62506]: DEBUG nova.compute.manager [-] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2077.748712] env[62506]: DEBUG nova.network.neutron [-] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2077.763643] env[62506]: DEBUG nova.network.neutron [-] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2077.915095] env[62506]: DEBUG nova.scheduler.client.report [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2078.265778] env[62506]: DEBUG nova.network.neutron [-] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2078.421299] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.790s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2078.422048] env[62506]: ERROR nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Traceback (most recent call last): [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self.driver.spawn(context, instance, image_meta, [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] vm_ref = self.build_virtual_machine(instance, [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] vif_infos = vmwarevif.get_vif_info(self._session, [ 2078.422048] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] for vif in network_info: [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] return self._sync_wrapper(fn, *args, **kwargs) [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self.wait() [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self[:] = self._gt.wait() [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] return self._exit_event.wait() [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] current.throw(*self._exc) [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2078.422423] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] result = function(*args, **kwargs) [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] return func(*args, **kwargs) [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] raise e [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] nwinfo = self.network_api.allocate_for_instance( [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] created_port_ids = self._update_ports_for_instance( [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] with excutils.save_and_reraise_exception(): [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] self.force_reraise() [ 2078.422740] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] raise self.value [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] updated_port = self._update_port( [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] _ensure_no_port_binding_failure(port) [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] raise exception.PortBindingFailed(port_id=port['id']) [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] nova.exception.PortBindingFailed: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. [ 2078.423101] env[62506]: ERROR nova.compute.manager [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] [ 2078.423101] env[62506]: DEBUG nova.compute.utils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2078.424182] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 15.042s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2078.425764] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Build of instance 7f33e930-8252-4019-81e1-79329a1ee1e3 was re-scheduled: Binding failed for port 8717740a-13c8-40c4-b3e3-6764966e51d4, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2078.427462] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2078.427462] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquiring lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2078.427462] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Acquired lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2078.427462] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2078.768071] env[62506]: INFO nova.compute.manager [-] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Took 1.02 seconds to deallocate network for instance. [ 2078.770327] env[62506]: DEBUG nova.compute.claims [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2078.770547] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2078.775426] env[62506]: INFO nova.scheduler.client.report [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Deleted allocations for instance 73ff1d78-5841-4532-a3f0-51e3f8e3314e [ 2078.959348] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2079.083335] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2079.283857] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c61fd5fa-d449-4273-bca6-c04208f21754 tempest-VolumesAdminNegativeTest-852963336 tempest-VolumesAdminNegativeTest-852963336-project-member] Lock "73ff1d78-5841-4532-a3f0-51e3f8e3314e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 100.334s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2079.471526] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 5e16bc57-424c-45b0-8e85-1d80bac92b18 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2079.471719] env[62506]: WARNING nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8665bdf3-3930-4f45-b5be-e308f8f53828 is not being actively managed by this compute host but has allocations referencing this compute host: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. Skipping heal of allocation because we do not know what to do. [ 2079.589810] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Releasing lock "refresh_cache-7f33e930-8252-4019-81e1-79329a1ee1e3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2079.589810] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2079.589810] env[62506]: DEBUG nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2079.589810] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2079.618643] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2079.786558] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2079.974773] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7f33e930-8252-4019-81e1-79329a1ee1e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2079.975034] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b9edd77e-2f5a-4050-aaeb-c32b67d9aef7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2079.975242] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 56c46b60-5feb-4b7a-8c4c-b010f4cf7009 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2079.975428] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 635dfc2e-d53a-467a-a23e-ee927ff5b2b1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2079.975609] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 11614111-9445-4299-8626-d65d2038b0cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2079.975795] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance bd441dae-ac8b-41bd-8fae-15dca58f820e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2080.121623] env[62506]: DEBUG nova.network.neutron [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2080.311292] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2080.478914] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance b45f2334-c604-4454-993e-e1c16a6c1ef5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2080.628734] env[62506]: INFO nova.compute.manager [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] [instance: 7f33e930-8252-4019-81e1-79329a1ee1e3] Took 1.04 seconds to deallocate network for instance. [ 2080.982551] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 4ec5b60e-a35c-40b5-bf8b-167b9d856f98 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2081.486300] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 94a23198-2d64-4003-93cc-93522fe70172 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2081.665483] env[62506]: INFO nova.scheduler.client.report [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Deleted allocations for instance 7f33e930-8252-4019-81e1-79329a1ee1e3 [ 2081.989224] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance caa77bc8-f831-418e-bb46-bd7d5ed6ef2f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2082.174274] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afcd2d9c-0a07-45ce-87cf-c18c07648c76 tempest-ServerTagsTestJSON-1439254211 tempest-ServerTagsTestJSON-1439254211-project-member] Lock "7f33e930-8252-4019-81e1-79329a1ee1e3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 89.902s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2082.495620] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 6d89deb7-cc37-4feb-a998-fef1d8a92947 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2082.680831] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2083.000032] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 91925f54-176b-4f34-8634-46bf7cb73bdd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2083.208430] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2083.502275] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 80c76c85-2575-4733-a86a-111d138d69d4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2084.006739] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 1147abf9-da6d-4e51-878e-b8dc9a3f6141 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2084.509806] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance d4e49d27-403b-4f85-95b7-ae07e9bb07b8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2084.510091] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2084.510245] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2084.721646] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5be977b-7843-4028-b182-609ba853503c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2084.729301] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c642407a-d215-44d4-bd5d-518e019365d5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2084.758644] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c123c5d2-9ebb-4abc-a9dd-c699f1714c24 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2084.766186] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-141da628-6015-4d0a-be73-c272ef162f8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2084.781492] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2085.217891] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2085.218150] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2085.288679] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2085.795699] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2085.795699] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 7.371s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2085.795699] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.900s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2086.517169] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db3e5e01-c33d-47a1-a69b-d837cc590f88 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2086.530197] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1640f3e-9026-4a72-a7c1-06c34a41cac7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2086.563956] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f551558-e4ac-4945-b996-e880d92ab380 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2086.571902] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0648b56-1cad-4e1a-98c1-74df25a70c1c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2086.586782] env[62506]: DEBUG nova.compute.provider_tree [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2087.090089] env[62506]: DEBUG nova.scheduler.client.report [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2087.595206] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.800s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2087.595954] env[62506]: ERROR nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Traceback (most recent call last): [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self.driver.spawn(context, instance, image_meta, [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] vm_ref = self.build_virtual_machine(instance, [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] vif_infos = vmwarevif.get_vif_info(self._session, [ 2087.595954] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] for vif in network_info: [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] return self._sync_wrapper(fn, *args, **kwargs) [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self.wait() [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self[:] = self._gt.wait() [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] return self._exit_event.wait() [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] current.throw(*self._exc) [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2087.596322] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] result = function(*args, **kwargs) [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] return func(*args, **kwargs) [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] raise e [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] nwinfo = self.network_api.allocate_for_instance( [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] created_port_ids = self._update_ports_for_instance( [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] with excutils.save_and_reraise_exception(): [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] self.force_reraise() [ 2087.596746] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] raise self.value [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] updated_port = self._update_port( [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] _ensure_no_port_binding_failure(port) [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] raise exception.PortBindingFailed(port_id=port['id']) [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] nova.exception.PortBindingFailed: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. [ 2087.597112] env[62506]: ERROR nova.compute.manager [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] [ 2087.597112] env[62506]: DEBUG nova.compute.utils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2087.597941] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 22.096s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2087.598149] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2087.600442] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.381s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2087.603316] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Build of instance b9edd77e-2f5a-4050-aaeb-c32b67d9aef7 was re-scheduled: Binding failed for port 870bc4bc-821e-4f63-97e0-726729147716, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2087.603751] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2087.603967] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquiring lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2087.604129] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Acquired lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2087.604296] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2087.620450] env[62506]: INFO nova.scheduler.client.report [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleted allocations for instance 8665bdf3-3930-4f45-b5be-e308f8f53828 [ 2087.798567] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2087.798818] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2087.971441] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "ec283c13-1c5f-4dff-9254-3c278d87935a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2087.971677] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "ec283c13-1c5f-4dff-9254-3c278d87935a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2088.127807] env[62506]: DEBUG oslo_concurrency.lockutils [None req-26e9f0f9-9e68-4fd9-b988-6899f0f79457 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "8665bdf3-3930-4f45-b5be-e308f8f53828" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 26.812s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2088.170791] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2088.251944] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2088.303881] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2088.304090] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 2088.304222] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 2088.334607] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-203d3fb6-08b7-4ce4-bfe0-210ed2daadd0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2088.342167] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17370c3c-7336-48b5-a613-7920ff94a742 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2088.373547] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ba3ef83-c013-4f77-a43e-37a4ccd37dea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2088.380906] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b832e1ff-43cf-44ce-b729-ef14167410eb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2088.393888] env[62506]: DEBUG nova.compute.provider_tree [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2088.754359] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Releasing lock "refresh_cache-b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2088.754607] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2088.754792] env[62506]: DEBUG nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2088.754960] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2088.771467] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2088.810651] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2088.810828] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2088.810959] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2088.875239] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "refresh_cache-5e16bc57-424c-45b0-8e85-1d80bac92b18" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2088.875398] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquired lock "refresh_cache-5e16bc57-424c-45b0-8e85-1d80bac92b18" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2088.876020] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Forcefully refreshing network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2061}} [ 2088.876020] env[62506]: DEBUG nova.objects.instance [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lazy-loading 'info_cache' on Instance uuid 5e16bc57-424c-45b0-8e85-1d80bac92b18 {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 2088.896465] env[62506]: DEBUG nova.scheduler.client.report [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2089.254023] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "5e16bc57-424c-45b0-8e85-1d80bac92b18" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2089.254331] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "5e16bc57-424c-45b0-8e85-1d80bac92b18" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2089.254731] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "5e16bc57-424c-45b0-8e85-1d80bac92b18-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2089.254948] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "5e16bc57-424c-45b0-8e85-1d80bac92b18-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2089.255145] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "5e16bc57-424c-45b0-8e85-1d80bac92b18-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2089.257129] env[62506]: INFO nova.compute.manager [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Terminating instance [ 2089.274361] env[62506]: DEBUG nova.network.neutron [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2089.402875] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.802s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2089.403536] env[62506]: ERROR nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Traceback (most recent call last): [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self.driver.spawn(context, instance, image_meta, [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] vm_ref = self.build_virtual_machine(instance, [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] vif_infos = vmwarevif.get_vif_info(self._session, [ 2089.403536] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] for vif in network_info: [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] return self._sync_wrapper(fn, *args, **kwargs) [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self.wait() [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self[:] = self._gt.wait() [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] return self._exit_event.wait() [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] current.throw(*self._exc) [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2089.403849] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] result = function(*args, **kwargs) [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] return func(*args, **kwargs) [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] raise e [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] nwinfo = self.network_api.allocate_for_instance( [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] created_port_ids = self._update_ports_for_instance( [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] with excutils.save_and_reraise_exception(): [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] self.force_reraise() [ 2089.404163] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] raise self.value [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] updated_port = self._update_port( [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] _ensure_no_port_binding_failure(port) [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] raise exception.PortBindingFailed(port_id=port['id']) [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] nova.exception.PortBindingFailed: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. [ 2089.404496] env[62506]: ERROR nova.compute.manager [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] [ 2089.404496] env[62506]: DEBUG nova.compute.utils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2089.405922] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Build of instance 56c46b60-5feb-4b7a-8c4c-b010f4cf7009 was re-scheduled: Binding failed for port 74764185-25a6-4229-9d2c-9858985d8956, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2089.406348] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2089.406615] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2089.406720] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2089.406879] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2089.407787] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.392s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2089.409189] env[62506]: INFO nova.compute.claims [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2089.760538] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "refresh_cache-5e16bc57-424c-45b0-8e85-1d80bac92b18" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2089.776313] env[62506]: INFO nova.compute.manager [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] [instance: b9edd77e-2f5a-4050-aaeb-c32b67d9aef7] Took 1.02 seconds to deallocate network for instance. [ 2089.897043] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2089.937680] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2090.040304] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2090.445109] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2090.542952] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-56c46b60-5feb-4b7a-8c4c-b010f4cf7009" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2090.543341] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2090.543492] env[62506]: DEBUG nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2090.543664] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2090.560711] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2090.608488] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cd24903-e867-4019-8d1d-484da396cb6b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2090.617046] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f6fb3c4-3335-4e52-9a41-4a88ae2c4805 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2090.645757] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd919181-3e25-4e3d-9ac3-9ab7ded6ee5f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2090.653086] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b008c25-e625-4d53-9899-903dd8de5314 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2090.666361] env[62506]: DEBUG nova.compute.provider_tree [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2090.802702] env[62506]: INFO nova.scheduler.client.report [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Deleted allocations for instance b9edd77e-2f5a-4050-aaeb-c32b67d9aef7 [ 2090.947255] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Releasing lock "refresh_cache-5e16bc57-424c-45b0-8e85-1d80bac92b18" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2090.947494] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Updated the network info_cache for instance {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10516}} [ 2090.947769] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquired lock "refresh_cache-5e16bc57-424c-45b0-8e85-1d80bac92b18" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2090.947939] env[62506]: DEBUG nova.network.neutron [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2090.949054] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2090.949290] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2090.949388] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2090.949549] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2090.949677] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 2091.062887] env[62506]: DEBUG nova.network.neutron [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2091.169120] env[62506]: DEBUG nova.scheduler.client.report [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2091.311478] env[62506]: DEBUG oslo_concurrency.lockutils [None req-8a434e4a-66f4-4343-ade1-e12132b006e9 tempest-AttachVolumeShelveTestJSON-967368784 tempest-AttachVolumeShelveTestJSON-967368784-project-member] Lock "b9edd77e-2f5a-4050-aaeb-c32b67d9aef7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 91.808s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2091.472025] env[62506]: DEBUG nova.network.neutron [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2091.559863] env[62506]: DEBUG nova.network.neutron [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2091.565400] env[62506]: INFO nova.compute.manager [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 56c46b60-5feb-4b7a-8c4c-b010f4cf7009] Took 1.02 seconds to deallocate network for instance. [ 2091.674498] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.267s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2091.675142] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2091.680863] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.251s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2091.817642] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2092.064092] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Releasing lock "refresh_cache-5e16bc57-424c-45b0-8e85-1d80bac92b18" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2092.064589] env[62506]: DEBUG nova.compute.manager [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2092.064795] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2092.066742] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d870b279-a8f4-449d-8118-1372859e7649 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.080124] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2092.080580] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-afad693a-a685-4190-8a16-74b8bf835707 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.086892] env[62506]: DEBUG oslo_vmware.api [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2092.086892] env[62506]: value = "task-2190862" [ 2092.086892] env[62506]: _type = "Task" [ 2092.086892] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2092.098431] env[62506]: DEBUG oslo_vmware.api [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190862, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2092.182483] env[62506]: DEBUG nova.compute.utils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2092.184273] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2092.184466] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2092.263653] env[62506]: DEBUG nova.policy [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '69c10cca15644239bf7b17e65d48c658', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '906a5d81ca404211b5a28f8683248ef1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2092.342021] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2092.398165] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d22dae6-260d-493c-97fd-3c994ef552ee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.406178] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29b6dbd3-fed1-4364-94b2-817774bc3fae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.441678] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d57f0f29-e781-4c86-91f4-2a46c327fad4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.449704] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-871e85a5-6152-4e02-b6f2-405df2bebdab {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.464737] env[62506]: DEBUG nova.compute.provider_tree [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2092.597422] env[62506]: DEBUG oslo_vmware.api [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190862, 'name': PowerOffVM_Task, 'duration_secs': 0.112316} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2092.597755] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2092.597849] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2092.598186] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-6d48e244-5ae3-44d8-97ca-7659ce54b840 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.611148] env[62506]: INFO nova.scheduler.client.report [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Deleted allocations for instance 56c46b60-5feb-4b7a-8c4c-b010f4cf7009 [ 2092.674020] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2092.674270] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2092.674453] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleting the datastore file [datastore2] 5e16bc57-424c-45b0-8e85-1d80bac92b18 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2092.674733] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a0b76256-ce2d-44a0-871c-a639d40d1ece {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2092.681737] env[62506]: DEBUG oslo_vmware.api [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for the task: (returnval){ [ 2092.681737] env[62506]: value = "task-2190864" [ 2092.681737] env[62506]: _type = "Task" [ 2092.681737] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2092.689751] env[62506]: DEBUG oslo_vmware.api [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190864, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2092.691377] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2092.847032] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Successfully created port: 68eed6aa-a43d-4488-96ec-b81369cc171a {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2092.968533] env[62506]: DEBUG nova.scheduler.client.report [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2093.120282] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2007a464-dccc-4c80-839e-9a438b040e7f tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "56c46b60-5feb-4b7a-8c4c-b010f4cf7009" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 80.879s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2093.193260] env[62506]: DEBUG oslo_vmware.api [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Task: {'id': task-2190864, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.20812} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2093.193564] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2093.193757] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2093.193938] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2093.194126] env[62506]: INFO nova.compute.manager [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Took 1.13 seconds to destroy the instance on the hypervisor. [ 2093.194371] env[62506]: DEBUG oslo.service.loopingcall [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2093.194568] env[62506]: DEBUG nova.compute.manager [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2093.194671] env[62506]: DEBUG nova.network.neutron [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2093.216374] env[62506]: DEBUG nova.network.neutron [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2093.477286] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.794s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2093.477286] env[62506]: ERROR nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Traceback (most recent call last): [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self.driver.spawn(context, instance, image_meta, [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2093.477286] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] vm_ref = self.build_virtual_machine(instance, [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] vif_infos = vmwarevif.get_vif_info(self._session, [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] for vif in network_info: [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] return self._sync_wrapper(fn, *args, **kwargs) [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self.wait() [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self[:] = self._gt.wait() [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] return self._exit_event.wait() [ 2093.477613] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] current.throw(*self._exc) [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] result = function(*args, **kwargs) [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] return func(*args, **kwargs) [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] raise e [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] nwinfo = self.network_api.allocate_for_instance( [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] created_port_ids = self._update_ports_for_instance( [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2093.477975] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] with excutils.save_and_reraise_exception(): [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] self.force_reraise() [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] raise self.value [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] updated_port = self._update_port( [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] _ensure_no_port_binding_failure(port) [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] raise exception.PortBindingFailed(port_id=port['id']) [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] nova.exception.PortBindingFailed: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. [ 2093.478345] env[62506]: ERROR nova.compute.manager [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] [ 2093.478681] env[62506]: DEBUG nova.compute.utils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2093.478681] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.814s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2093.486130] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Build of instance 635dfc2e-d53a-467a-a23e-ee927ff5b2b1 was re-scheduled: Binding failed for port a57b4361-4fee-4c7d-b32a-75abd59ec9ca, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2093.486130] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2093.487544] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2093.487863] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquired lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2093.488167] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2093.622942] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2093.700995] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2093.719129] env[62506]: DEBUG nova.network.neutron [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2093.724537] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:42Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='84',id=12,is_public=True,memory_mb=256,name='m1.micro',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2093.724783] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2093.725062] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2093.725540] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2093.725728] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2093.725888] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2093.726113] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2093.726273] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2093.726436] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2093.726591] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2093.726754] env[62506]: DEBUG nova.virt.hardware [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2093.727617] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8add04d-75d8-44ac-9945-42b4e7aad817 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2093.736853] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e284ec27-12ec-4f0b-bb68-6bdcc5cdef98 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2094.023789] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2094.090138] env[62506]: DEBUG nova.compute.manager [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Received event network-changed-68eed6aa-a43d-4488-96ec-b81369cc171a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2094.090466] env[62506]: DEBUG nova.compute.manager [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Refreshing instance network info cache due to event network-changed-68eed6aa-a43d-4488-96ec-b81369cc171a. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2094.090555] env[62506]: DEBUG oslo_concurrency.lockutils [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] Acquiring lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2094.090694] env[62506]: DEBUG oslo_concurrency.lockutils [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] Acquired lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2094.090849] env[62506]: DEBUG nova.network.neutron [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Refreshing network info cache for port 68eed6aa-a43d-4488-96ec-b81369cc171a {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2094.119892] env[62506]: ERROR nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2094.119892] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2094.119892] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2094.119892] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2094.119892] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2094.119892] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2094.119892] env[62506]: ERROR nova.compute.manager raise self.value [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2094.119892] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2094.119892] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2094.119892] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2094.120386] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2094.120386] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2094.120386] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2094.120386] env[62506]: ERROR nova.compute.manager [ 2094.120386] env[62506]: Traceback (most recent call last): [ 2094.120386] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2094.120386] env[62506]: listener.cb(fileno) [ 2094.120386] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2094.120386] env[62506]: result = function(*args, **kwargs) [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2094.120386] env[62506]: return func(*args, **kwargs) [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2094.120386] env[62506]: raise e [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2094.120386] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2094.120386] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2094.120386] env[62506]: with excutils.save_and_reraise_exception(): [ 2094.120386] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2094.120386] env[62506]: self.force_reraise() [ 2094.120386] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2094.120386] env[62506]: raise self.value [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2094.120386] env[62506]: updated_port = self._update_port( [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2094.120386] env[62506]: _ensure_no_port_binding_failure(port) [ 2094.120386] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2094.120386] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2094.121144] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2094.121144] env[62506]: Removing descriptor: 19 [ 2094.121200] env[62506]: ERROR nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Traceback (most recent call last): [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] yield resources [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self.driver.spawn(context, instance, image_meta, [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] vm_ref = self.build_virtual_machine(instance, [ 2094.121200] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] vif_infos = vmwarevif.get_vif_info(self._session, [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] for vif in network_info: [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return self._sync_wrapper(fn, *args, **kwargs) [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self.wait() [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self[:] = self._gt.wait() [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return self._exit_event.wait() [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2094.121459] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] result = hub.switch() [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return self.greenlet.switch() [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] result = function(*args, **kwargs) [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return func(*args, **kwargs) [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] raise e [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] nwinfo = self.network_api.allocate_for_instance( [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] created_port_ids = self._update_ports_for_instance( [ 2094.121830] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] with excutils.save_and_reraise_exception(): [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self.force_reraise() [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] raise self.value [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] updated_port = self._update_port( [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] _ensure_no_port_binding_failure(port) [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] raise exception.PortBindingFailed(port_id=port['id']) [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2094.122147] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] [ 2094.122533] env[62506]: INFO nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Terminating instance [ 2094.130506] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2094.158218] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2094.221192] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fed7e3e8-977d-4b28-ad06-a7832154b6c0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2094.229084] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8efb37f-0b93-4255-932d-fa9507e125e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2094.232783] env[62506]: INFO nova.compute.manager [-] [instance: 5e16bc57-424c-45b0-8e85-1d80bac92b18] Took 1.04 seconds to deallocate network for instance. [ 2094.268970] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-953f66f6-8aac-4ebd-b534-5b4573062b61 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2094.277352] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07fcdef7-c60b-4996-a63f-abdf300bc114 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2094.290945] env[62506]: DEBUG nova.compute.provider_tree [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2094.501475] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "ed482516-f639-4f5d-8e79-cdb72c85e304" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2094.501717] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "ed482516-f639-4f5d-8e79-cdb72c85e304" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2094.614502] env[62506]: DEBUG nova.network.neutron [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2094.632421] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2094.633957] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Releasing lock "refresh_cache-635dfc2e-d53a-467a-a23e-ee927ff5b2b1" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2094.634180] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2094.634368] env[62506]: DEBUG nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2094.634528] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2094.649690] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2094.695999] env[62506]: DEBUG nova.network.neutron [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2094.769545] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2094.811452] env[62506]: ERROR nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [req-74f1e036-9022-49f6-8059-54c00a7b0cc7] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-74f1e036-9022-49f6-8059-54c00a7b0cc7"}]}: nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2094.828269] env[62506]: DEBUG nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 2094.842621] env[62506]: DEBUG nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 2094.842853] env[62506]: DEBUG nova.compute.provider_tree [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2094.854098] env[62506]: DEBUG nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 2094.871377] env[62506]: DEBUG nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 2095.049150] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03ab1f2d-93cd-4ffa-92d6-19f116588612 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2095.056938] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acd85ff5-29ed-4a5a-b154-643f44b34e56 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2095.087319] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4136d38e-a9a8-4577-8230-929d12f73cda {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2095.094406] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2d4ade8-bab0-459b-9438-c2fe2dbb7fc1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2095.107820] env[62506]: DEBUG nova.compute.provider_tree [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2095.152068] env[62506]: DEBUG nova.network.neutron [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2095.198274] env[62506]: DEBUG oslo_concurrency.lockutils [req-1c6c7427-5fd5-4a6d-a991-6eca11851fe2 req-98ba7f3f-f06a-4386-affc-e48b40dede06 service nova] Releasing lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2095.198672] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquired lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2095.198859] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2095.639699] env[62506]: DEBUG nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 88 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 2095.639976] env[62506]: DEBUG nova.compute.provider_tree [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 88 to 89 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 2095.640175] env[62506]: DEBUG nova.compute.provider_tree [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2095.654786] env[62506]: INFO nova.compute.manager [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 635dfc2e-d53a-467a-a23e-ee927ff5b2b1] Took 1.02 seconds to deallocate network for instance. [ 2095.715911] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2095.785295] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2096.114349] env[62506]: DEBUG nova.compute.manager [req-1c02bfc4-2cad-47c3-9393-ae3477fe26c2 req-73710a47-ae3a-4fba-b401-c8da85136396 service nova] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Received event network-vif-deleted-68eed6aa-a43d-4488-96ec-b81369cc171a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2096.145534] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.668s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2096.146261] env[62506]: ERROR nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] Traceback (most recent call last): [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self.driver.spawn(context, instance, image_meta, [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] vm_ref = self.build_virtual_machine(instance, [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2096.146261] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] for vif in network_info: [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] return self._sync_wrapper(fn, *args, **kwargs) [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self.wait() [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self[:] = self._gt.wait() [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] return self._exit_event.wait() [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] current.throw(*self._exc) [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2096.146588] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] result = function(*args, **kwargs) [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] return func(*args, **kwargs) [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] raise e [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] nwinfo = self.network_api.allocate_for_instance( [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] created_port_ids = self._update_ports_for_instance( [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] with excutils.save_and_reraise_exception(): [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] self.force_reraise() [ 2096.146979] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] raise self.value [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] updated_port = self._update_port( [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] _ensure_no_port_binding_failure(port) [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] raise exception.PortBindingFailed(port_id=port['id']) [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] nova.exception.PortBindingFailed: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. [ 2096.147352] env[62506]: ERROR nova.compute.manager [instance: 11614111-9445-4299-8626-d65d2038b0cd] [ 2096.147352] env[62506]: DEBUG nova.compute.utils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2096.148066] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.703s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2096.149873] env[62506]: INFO nova.compute.claims [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2096.158038] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Build of instance 11614111-9445-4299-8626-d65d2038b0cd was re-scheduled: Binding failed for port 6b70fa38-c05c-476f-8129-d1adeca8a94d, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2096.158485] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2096.158720] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2096.158871] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquired lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2096.159044] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2096.287665] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Releasing lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2096.288270] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2096.288509] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2096.288818] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9c0e251e-f08f-442a-b531-dbfb5fc289bf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2096.299027] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99e3ede8-c98d-4d28-aa13-330b76da18df {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2096.319883] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b45f2334-c604-4454-993e-e1c16a6c1ef5 could not be found. [ 2096.320110] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2096.320302] env[62506]: INFO nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2096.320541] env[62506]: DEBUG oslo.service.loopingcall [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2096.320755] env[62506]: DEBUG nova.compute.manager [-] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2096.320850] env[62506]: DEBUG nova.network.neutron [-] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2096.334790] env[62506]: DEBUG nova.network.neutron [-] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2096.681181] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2096.691236] env[62506]: INFO nova.scheduler.client.report [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Deleted allocations for instance 635dfc2e-d53a-467a-a23e-ee927ff5b2b1 [ 2096.761581] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2096.837831] env[62506]: DEBUG nova.network.neutron [-] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2097.199201] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f6ef946b-9329-40f1-90a5-c97086950a67 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "635dfc2e-d53a-467a-a23e-ee927ff5b2b1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 83.345s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2097.265176] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Releasing lock "refresh_cache-11614111-9445-4299-8626-d65d2038b0cd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2097.265433] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2097.265599] env[62506]: DEBUG nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2097.265838] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2097.282022] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2097.340497] env[62506]: INFO nova.compute.manager [-] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Took 1.02 seconds to deallocate network for instance. [ 2097.342752] env[62506]: DEBUG nova.compute.claims [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2097.342969] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2097.347008] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6babc06-86bc-4812-8301-1aaf6a07de16 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2097.354925] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74358fd3-a089-4522-bad5-ab2603175372 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2097.383936] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f924998d-689b-49fd-8953-f4e543673bc8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2097.391115] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89de68ab-179c-480f-90ac-e8b428b488b4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2097.404976] env[62506]: DEBUG nova.compute.provider_tree [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2097.704675] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2097.784189] env[62506]: DEBUG nova.network.neutron [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2097.907886] env[62506]: DEBUG nova.scheduler.client.report [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2098.222629] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2098.286453] env[62506]: INFO nova.compute.manager [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: 11614111-9445-4299-8626-d65d2038b0cd] Took 1.02 seconds to deallocate network for instance. [ 2098.412736] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.264s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2098.413298] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2098.415880] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.190s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2098.417758] env[62506]: INFO nova.compute.claims [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2098.922592] env[62506]: DEBUG nova.compute.utils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2098.927042] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2098.927185] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2098.971140] env[62506]: DEBUG nova.policy [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3c9c372663b34fcaa0dac3e1c657e044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fc10dc6a19fb41cdbf046f6da6dc69fe', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2099.250327] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Successfully created port: 1b0733dc-bfc2-427d-b62f-aed00409289f {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2099.318020] env[62506]: INFO nova.scheduler.client.report [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Deleted allocations for instance 11614111-9445-4299-8626-d65d2038b0cd [ 2099.428578] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2099.612730] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac7028e3-644d-4676-ab68-2a49c13eed86 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2099.620667] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02e96c30-02c0-4e49-94bc-c1addf6dd04c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2099.659060] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7fadce4-cc8d-4abc-828d-e9fac4d1b306 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2099.667047] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9e00322-68bd-49a9-95b6-39b09414b838 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2099.681237] env[62506]: DEBUG nova.compute.provider_tree [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2099.829288] env[62506]: DEBUG oslo_concurrency.lockutils [None req-87e87588-1b9e-4259-9a03-1c96639302d9 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "11614111-9445-4299-8626-d65d2038b0cd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.248s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2100.036290] env[62506]: DEBUG nova.compute.manager [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Received event network-changed-1b0733dc-bfc2-427d-b62f-aed00409289f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2100.036690] env[62506]: DEBUG nova.compute.manager [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Refreshing instance network info cache due to event network-changed-1b0733dc-bfc2-427d-b62f-aed00409289f. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2100.036747] env[62506]: DEBUG oslo_concurrency.lockutils [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] Acquiring lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2100.036876] env[62506]: DEBUG oslo_concurrency.lockutils [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] Acquired lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2100.037048] env[62506]: DEBUG nova.network.neutron [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Refreshing network info cache for port 1b0733dc-bfc2-427d-b62f-aed00409289f {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2100.184642] env[62506]: DEBUG nova.scheduler.client.report [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2100.198710] env[62506]: ERROR nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2100.198710] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2100.198710] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2100.198710] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2100.198710] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2100.198710] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2100.198710] env[62506]: ERROR nova.compute.manager raise self.value [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2100.198710] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2100.198710] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2100.198710] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2100.199288] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2100.199288] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2100.199288] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2100.199288] env[62506]: ERROR nova.compute.manager [ 2100.199288] env[62506]: Traceback (most recent call last): [ 2100.199288] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2100.199288] env[62506]: listener.cb(fileno) [ 2100.199288] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2100.199288] env[62506]: result = function(*args, **kwargs) [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2100.199288] env[62506]: return func(*args, **kwargs) [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2100.199288] env[62506]: raise e [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2100.199288] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2100.199288] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2100.199288] env[62506]: with excutils.save_and_reraise_exception(): [ 2100.199288] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2100.199288] env[62506]: self.force_reraise() [ 2100.199288] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2100.199288] env[62506]: raise self.value [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2100.199288] env[62506]: updated_port = self._update_port( [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2100.199288] env[62506]: _ensure_no_port_binding_failure(port) [ 2100.199288] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2100.199288] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2100.200096] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2100.200096] env[62506]: Removing descriptor: 20 [ 2100.331479] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2100.445988] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2100.472393] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2100.472940] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2100.472940] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2100.473069] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2100.473272] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2100.473467] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2100.473694] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2100.473855] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2100.474048] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2100.474523] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2100.474523] env[62506]: DEBUG nova.virt.hardware [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2100.475286] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7faf5b23-dfb3-415e-8b9a-eccc33e48b5e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2100.483268] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c628a4b7-a1e3-4abc-95e6-5b3b78fe16c0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2100.498139] env[62506]: ERROR nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Traceback (most recent call last): [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] yield resources [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self.driver.spawn(context, instance, image_meta, [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] vm_ref = self.build_virtual_machine(instance, [ 2100.498139] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] vif_infos = vmwarevif.get_vif_info(self._session, [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] for vif in network_info: [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] return self._sync_wrapper(fn, *args, **kwargs) [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self.wait() [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self[:] = self._gt.wait() [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] return self._exit_event.wait() [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2100.498459] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] current.throw(*self._exc) [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] result = function(*args, **kwargs) [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] return func(*args, **kwargs) [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] raise e [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] nwinfo = self.network_api.allocate_for_instance( [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] created_port_ids = self._update_ports_for_instance( [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] with excutils.save_and_reraise_exception(): [ 2100.498762] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self.force_reraise() [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] raise self.value [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] updated_port = self._update_port( [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] _ensure_no_port_binding_failure(port) [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] raise exception.PortBindingFailed(port_id=port['id']) [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2100.499419] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] [ 2100.499419] env[62506]: INFO nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Terminating instance [ 2100.556847] env[62506]: DEBUG nova.network.neutron [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2100.628638] env[62506]: DEBUG nova.network.neutron [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2100.689731] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.274s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2100.690278] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2100.693386] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.923s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2100.853412] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2101.002300] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2101.131571] env[62506]: DEBUG oslo_concurrency.lockutils [req-11a61636-e255-4028-8fc9-dc3be6db4cd6 req-93fb1d1c-62c9-4207-b42f-e0c7bae3db8a service nova] Releasing lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2101.132012] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2101.132219] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2101.197707] env[62506]: DEBUG nova.compute.utils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2101.202020] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2101.202202] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2101.248331] env[62506]: DEBUG nova.policy [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '854b5c4700d247b8aeef871aac9100fc', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c395dcc5b1d943e3b388cce1937c906c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2101.369144] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7975dd2-4a8b-4fdb-9997-ecf3f1ae8b4d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2101.377339] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61814cc4-c6a2-49de-b2bd-aed443591025 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2101.407500] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b65ded62-c5a1-4961-a491-1866025e72fb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2101.414977] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35c1f6b0-82f3-4b48-a0ce-f7377d45a61f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2101.429189] env[62506]: DEBUG nova.compute.provider_tree [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2101.553565] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Successfully created port: fe682248-e43e-417d-b795-828cfafe5d2e {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2101.610821] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2101.667433] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2101.703155] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2101.743784] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2101.934992] env[62506]: DEBUG nova.scheduler.client.report [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2102.089460] env[62506]: DEBUG nova.compute.manager [req-4b2696fc-9de7-4ba6-8580-1023dcfcd27b req-62338250-8530-4b6e-a848-374bdfe11e6a service nova] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Received event network-vif-deleted-1b0733dc-bfc2-427d-b62f-aed00409289f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2102.245956] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2102.246555] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2102.246630] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2102.247697] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-84aba0d7-9431-46a3-8d67-94dd8d8ddb73 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2102.256575] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e9e250c-b2ae-48bd-b7bd-3fc1750ba99f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2102.278884] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4ec5b60e-a35c-40b5-bf8b-167b9d856f98 could not be found. [ 2102.279117] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2102.279303] env[62506]: INFO nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2102.279545] env[62506]: DEBUG oslo.service.loopingcall [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2102.279771] env[62506]: DEBUG nova.compute.manager [-] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2102.279867] env[62506]: DEBUG nova.network.neutron [-] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2102.300191] env[62506]: DEBUG nova.network.neutron [-] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2102.345052] env[62506]: ERROR nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2102.345052] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2102.345052] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2102.345052] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2102.345052] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2102.345052] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2102.345052] env[62506]: ERROR nova.compute.manager raise self.value [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2102.345052] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2102.345052] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2102.345052] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2102.345505] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2102.345505] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2102.345505] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2102.345505] env[62506]: ERROR nova.compute.manager [ 2102.345505] env[62506]: Traceback (most recent call last): [ 2102.345505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2102.345505] env[62506]: listener.cb(fileno) [ 2102.345505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2102.345505] env[62506]: result = function(*args, **kwargs) [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2102.345505] env[62506]: return func(*args, **kwargs) [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2102.345505] env[62506]: raise e [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2102.345505] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2102.345505] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2102.345505] env[62506]: with excutils.save_and_reraise_exception(): [ 2102.345505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2102.345505] env[62506]: self.force_reraise() [ 2102.345505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2102.345505] env[62506]: raise self.value [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2102.345505] env[62506]: updated_port = self._update_port( [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2102.345505] env[62506]: _ensure_no_port_binding_failure(port) [ 2102.345505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2102.345505] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2102.346263] env[62506]: nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2102.346263] env[62506]: Removing descriptor: 20 [ 2102.440122] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.747s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2102.440776] env[62506]: ERROR nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Traceback (most recent call last): [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self.driver.spawn(context, instance, image_meta, [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] vm_ref = self.build_virtual_machine(instance, [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] vif_infos = vmwarevif.get_vif_info(self._session, [ 2102.440776] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] for vif in network_info: [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] return self._sync_wrapper(fn, *args, **kwargs) [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self.wait() [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self[:] = self._gt.wait() [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] return self._exit_event.wait() [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] current.throw(*self._exc) [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2102.441148] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] result = function(*args, **kwargs) [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] return func(*args, **kwargs) [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] raise e [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] nwinfo = self.network_api.allocate_for_instance( [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] created_port_ids = self._update_ports_for_instance( [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] with excutils.save_and_reraise_exception(): [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] self.force_reraise() [ 2102.441525] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] raise self.value [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] updated_port = self._update_port( [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] _ensure_no_port_binding_failure(port) [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] raise exception.PortBindingFailed(port_id=port['id']) [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] nova.exception.PortBindingFailed: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. [ 2102.441887] env[62506]: ERROR nova.compute.manager [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] [ 2102.441887] env[62506]: DEBUG nova.compute.utils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2102.442812] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.132s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2102.444667] env[62506]: INFO nova.compute.claims [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2102.447529] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Build of instance bd441dae-ac8b-41bd-8fae-15dca58f820e was re-scheduled: Binding failed for port b1c6ec19-23aa-47c5-bcba-b80f29b679a7, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2102.447973] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2102.448166] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquiring lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2102.448316] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Acquired lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2102.448473] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2102.714541] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2102.739970] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2102.740129] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2102.740515] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2102.740515] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2102.740603] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2102.740793] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2102.740922] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2102.741088] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2102.741267] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2102.741483] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2102.741616] env[62506]: DEBUG nova.virt.hardware [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2102.742851] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99f630f8-95cf-43c0-91e5-e72ba8ec783d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2102.751244] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4b4adde-d3b9-4d89-9f03-26f816fe89a4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2102.764935] env[62506]: ERROR nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] Traceback (most recent call last): [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] yield resources [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self.driver.spawn(context, instance, image_meta, [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] vm_ref = self.build_virtual_machine(instance, [ 2102.764935] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] vif_infos = vmwarevif.get_vif_info(self._session, [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] for vif in network_info: [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] return self._sync_wrapper(fn, *args, **kwargs) [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self.wait() [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self[:] = self._gt.wait() [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] return self._exit_event.wait() [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2102.765338] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] current.throw(*self._exc) [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] result = function(*args, **kwargs) [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] return func(*args, **kwargs) [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] raise e [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] nwinfo = self.network_api.allocate_for_instance( [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] created_port_ids = self._update_ports_for_instance( [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] with excutils.save_and_reraise_exception(): [ 2102.765713] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self.force_reraise() [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] raise self.value [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] updated_port = self._update_port( [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] _ensure_no_port_binding_failure(port) [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] raise exception.PortBindingFailed(port_id=port['id']) [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2102.766146] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] [ 2102.766146] env[62506]: INFO nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Terminating instance [ 2102.802636] env[62506]: DEBUG nova.network.neutron [-] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2102.967777] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2103.034409] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2103.269812] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquiring lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2103.270209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquired lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2103.270498] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2103.305259] env[62506]: INFO nova.compute.manager [-] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Took 1.03 seconds to deallocate network for instance. [ 2103.307549] env[62506]: DEBUG nova.compute.claims [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2103.307591] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2103.537875] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Releasing lock "refresh_cache-bd441dae-ac8b-41bd-8fae-15dca58f820e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2103.538133] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2103.538305] env[62506]: DEBUG nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2103.538470] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2103.554113] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2103.615195] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92695594-d6c2-476e-8950-8be5ab9dd6a5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2103.622981] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbe8c249-655a-47e7-b72f-eb30be79103a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2103.653186] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-074f008b-d748-4fa7-a118-06afd77dadf7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2103.660438] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0adb9ca3-6687-4bb9-8961-6e2adc777438 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2103.674071] env[62506]: DEBUG nova.compute.provider_tree [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2103.789123] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2103.859698] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2104.056938] env[62506]: DEBUG nova.network.neutron [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2104.113133] env[62506]: DEBUG nova.compute.manager [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Received event network-changed-fe682248-e43e-417d-b795-828cfafe5d2e {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2104.113351] env[62506]: DEBUG nova.compute.manager [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Refreshing instance network info cache due to event network-changed-fe682248-e43e-417d-b795-828cfafe5d2e. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2104.113554] env[62506]: DEBUG oslo_concurrency.lockutils [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] Acquiring lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2104.176723] env[62506]: DEBUG nova.scheduler.client.report [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2104.364642] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Releasing lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2104.365363] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2104.365699] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2104.366195] env[62506]: DEBUG oslo_concurrency.lockutils [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] Acquired lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2104.366503] env[62506]: DEBUG nova.network.neutron [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Refreshing network info cache for port fe682248-e43e-417d-b795-828cfafe5d2e {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2104.368462] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-498fd45b-2e00-4294-a443-d7851f26d13a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.379192] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2302804f-4b64-45eb-9eb5-336eac420ea8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.401580] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 94a23198-2d64-4003-93cc-93522fe70172 could not be found. [ 2104.401842] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2104.402073] env[62506]: INFO nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2104.402317] env[62506]: DEBUG oslo.service.loopingcall [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2104.402579] env[62506]: DEBUG nova.compute.manager [-] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2104.402672] env[62506]: DEBUG nova.network.neutron [-] [instance: 94a23198-2d64-4003-93cc-93522fe70172] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2104.418258] env[62506]: DEBUG nova.network.neutron [-] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2104.562297] env[62506]: INFO nova.compute.manager [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] [instance: bd441dae-ac8b-41bd-8fae-15dca58f820e] Took 1.02 seconds to deallocate network for instance. [ 2104.681515] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.239s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2104.683475] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.475s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2104.684686] env[62506]: INFO nova.compute.claims [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2104.888788] env[62506]: DEBUG nova.network.neutron [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2104.921127] env[62506]: DEBUG nova.network.neutron [-] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2104.958031] env[62506]: DEBUG nova.network.neutron [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2105.184546] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquiring lock "d6d9f272-8c5d-4885-b4b3-03e45b639e93" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2105.184832] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "d6d9f272-8c5d-4885-b4b3-03e45b639e93" acquired by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2105.424058] env[62506]: INFO nova.compute.manager [-] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Took 1.02 seconds to deallocate network for instance. [ 2105.426810] env[62506]: DEBUG nova.compute.claims [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2105.427087] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2105.460360] env[62506]: DEBUG oslo_concurrency.lockutils [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] Releasing lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2105.460624] env[62506]: DEBUG nova.compute.manager [req-50fa732d-2183-44b8-9f4e-572d3edb135b req-63ac45d4-c24b-40e0-98e0-db18861de515 service nova] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Received event network-vif-deleted-fe682248-e43e-417d-b795-828cfafe5d2e {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2105.593243] env[62506]: INFO nova.scheduler.client.report [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Deleted allocations for instance bd441dae-ac8b-41bd-8fae-15dca58f820e [ 2105.687408] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "d6d9f272-8c5d-4885-b4b3-03e45b639e93" "released" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: held 0.502s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2105.688078] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2105.865773] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2862e8aa-3c0e-4d09-b401-37d6c8ceed44 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2105.873673] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b84466d-3004-43ea-9cec-ada49787c9e0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2105.903431] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-431f75d2-4aa7-457b-9cc8-901a2746b218 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2105.910828] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ab4d5b5-f4e7-4515-a6da-092851c35d4f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2105.924043] env[62506]: DEBUG nova.compute.provider_tree [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2106.103350] env[62506]: DEBUG oslo_concurrency.lockutils [None req-200d949e-9563-4a63-9658-db482b725d90 tempest-MigrationsAdminTest-1665870808 tempest-MigrationsAdminTest-1665870808-project-member] Lock "bd441dae-ac8b-41bd-8fae-15dca58f820e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 91.216s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2106.196327] env[62506]: DEBUG nova.compute.utils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2106.197768] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2106.197989] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2106.248233] env[62506]: DEBUG nova.policy [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c145e0eaf70140f392c69069382b3c54', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9b83497081d446038907415265468d87', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2106.428834] env[62506]: DEBUG nova.scheduler.client.report [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2106.552017] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Successfully created port: 2b476cfb-59b3-4836-af68-f470d7b7c994 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2106.606601] env[62506]: DEBUG nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2106.700891] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2106.933869] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.250s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2106.934842] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2106.938722] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.598s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2106.939263] env[62506]: INFO nova.compute.claims [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2107.142449] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2107.321732] env[62506]: DEBUG nova.compute.manager [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Received event network-changed-2b476cfb-59b3-4836-af68-f470d7b7c994 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2107.321859] env[62506]: DEBUG nova.compute.manager [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Refreshing instance network info cache due to event network-changed-2b476cfb-59b3-4836-af68-f470d7b7c994. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2107.322485] env[62506]: DEBUG oslo_concurrency.lockutils [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] Acquiring lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2107.322485] env[62506]: DEBUG oslo_concurrency.lockutils [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] Acquired lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2107.322485] env[62506]: DEBUG nova.network.neutron [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Refreshing network info cache for port 2b476cfb-59b3-4836-af68-f470d7b7c994 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2107.447150] env[62506]: DEBUG nova.compute.utils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2107.448480] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2107.448692] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2107.476518] env[62506]: ERROR nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2107.476518] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2107.476518] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2107.476518] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2107.476518] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2107.476518] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2107.476518] env[62506]: ERROR nova.compute.manager raise self.value [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2107.476518] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2107.476518] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2107.476518] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2107.477066] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2107.477066] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2107.477066] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2107.477066] env[62506]: ERROR nova.compute.manager [ 2107.477066] env[62506]: Traceback (most recent call last): [ 2107.477066] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2107.477066] env[62506]: listener.cb(fileno) [ 2107.477066] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2107.477066] env[62506]: result = function(*args, **kwargs) [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2107.477066] env[62506]: return func(*args, **kwargs) [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2107.477066] env[62506]: raise e [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2107.477066] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2107.477066] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2107.477066] env[62506]: with excutils.save_and_reraise_exception(): [ 2107.477066] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2107.477066] env[62506]: self.force_reraise() [ 2107.477066] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2107.477066] env[62506]: raise self.value [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2107.477066] env[62506]: updated_port = self._update_port( [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2107.477066] env[62506]: _ensure_no_port_binding_failure(port) [ 2107.477066] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2107.477066] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2107.477929] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2107.477929] env[62506]: Removing descriptor: 20 [ 2107.517242] env[62506]: DEBUG nova.policy [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8615160e68af4fcf9e9cd0d9f97a0b3e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0e4a3d4707b74087b11ee299cd02976c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2107.715098] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2107.749723] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2107.750270] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2107.750270] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2107.751222] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2107.751415] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2107.751578] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2107.751798] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2107.751960] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2107.752378] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2107.752378] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2107.752549] env[62506]: DEBUG nova.virt.hardware [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2107.753444] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed263578-97f7-4e31-bc43-381b95c3e284 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2107.764460] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3d6b398-b533-4cdd-a1df-9b2692f210cd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2107.780048] env[62506]: ERROR nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Traceback (most recent call last): [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] yield resources [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self.driver.spawn(context, instance, image_meta, [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] vm_ref = self.build_virtual_machine(instance, [ 2107.780048] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] vif_infos = vmwarevif.get_vif_info(self._session, [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] for vif in network_info: [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] return self._sync_wrapper(fn, *args, **kwargs) [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self.wait() [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self[:] = self._gt.wait() [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] return self._exit_event.wait() [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2107.780369] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] current.throw(*self._exc) [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] result = function(*args, **kwargs) [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] return func(*args, **kwargs) [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] raise e [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] nwinfo = self.network_api.allocate_for_instance( [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] created_port_ids = self._update_ports_for_instance( [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] with excutils.save_and_reraise_exception(): [ 2107.780762] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self.force_reraise() [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] raise self.value [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] updated_port = self._update_port( [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] _ensure_no_port_binding_failure(port) [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] raise exception.PortBindingFailed(port_id=port['id']) [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2107.781128] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] [ 2107.781128] env[62506]: INFO nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Terminating instance [ 2107.842960] env[62506]: DEBUG nova.network.neutron [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2107.934681] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Successfully created port: 15763fc5-3cde-4c99-84fe-bc6a7747d33a {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2107.939859] env[62506]: DEBUG nova.network.neutron [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2107.951793] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2108.176281] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17189f2b-072d-4a65-b816-2a41c3a6c4a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2108.185556] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e102524-320c-43f2-8375-ad1dd6ad54e0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2108.217469] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daae6fef-ce07-430c-a8bd-dd3d0c5e3447 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2108.225144] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a0636c1-c871-413f-aae7-747601d68990 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2108.238852] env[62506]: DEBUG nova.compute.provider_tree [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2108.283972] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquiring lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2108.442897] env[62506]: DEBUG oslo_concurrency.lockutils [req-82f17f6c-77c3-4fe0-8fa5-5481d5d8a474 req-2518c09f-f2b4-4789-a9eb-2ada28f8b2f5 service nova] Releasing lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2108.443354] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquired lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2108.443570] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2108.743237] env[62506]: DEBUG nova.scheduler.client.report [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2108.798984] env[62506]: DEBUG nova.compute.manager [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Received event network-changed-15763fc5-3cde-4c99-84fe-bc6a7747d33a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2108.799220] env[62506]: DEBUG nova.compute.manager [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Refreshing instance network info cache due to event network-changed-15763fc5-3cde-4c99-84fe-bc6a7747d33a. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2108.799441] env[62506]: DEBUG oslo_concurrency.lockutils [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] Acquiring lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2108.799602] env[62506]: DEBUG oslo_concurrency.lockutils [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] Acquired lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2108.799798] env[62506]: DEBUG nova.network.neutron [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Refreshing network info cache for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2108.961126] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2108.976487] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2108.988139] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2108.988139] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2108.988139] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2108.988398] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2108.988474] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2108.988806] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2108.989072] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2108.989239] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2108.989446] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2108.989711] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2108.989919] env[62506]: DEBUG nova.virt.hardware [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2108.990838] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a3c1881-b4a5-4b9b-80e3-25c7c41b87da {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2108.993968] env[62506]: ERROR nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2108.993968] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2108.993968] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2108.993968] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2108.993968] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2108.993968] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2108.993968] env[62506]: ERROR nova.compute.manager raise self.value [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2108.993968] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2108.993968] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2108.993968] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2108.994479] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2108.994479] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2108.994479] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2108.994479] env[62506]: ERROR nova.compute.manager [ 2108.994479] env[62506]: Traceback (most recent call last): [ 2108.994479] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2108.994479] env[62506]: listener.cb(fileno) [ 2108.994479] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2108.994479] env[62506]: result = function(*args, **kwargs) [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2108.994479] env[62506]: return func(*args, **kwargs) [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2108.994479] env[62506]: raise e [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2108.994479] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2108.994479] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2108.994479] env[62506]: with excutils.save_and_reraise_exception(): [ 2108.994479] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2108.994479] env[62506]: self.force_reraise() [ 2108.994479] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2108.994479] env[62506]: raise self.value [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2108.994479] env[62506]: updated_port = self._update_port( [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2108.994479] env[62506]: _ensure_no_port_binding_failure(port) [ 2108.994479] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2108.994479] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2108.995359] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2108.995359] env[62506]: Removing descriptor: 19 [ 2109.003277] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74199ae9-09be-4ec6-9249-a4e6792d87c9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2109.015233] env[62506]: ERROR nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Traceback (most recent call last): [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] yield resources [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self.driver.spawn(context, instance, image_meta, [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] vm_ref = self.build_virtual_machine(instance, [ 2109.015233] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] vif_infos = vmwarevif.get_vif_info(self._session, [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] for vif in network_info: [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] return self._sync_wrapper(fn, *args, **kwargs) [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self.wait() [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self[:] = self._gt.wait() [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] return self._exit_event.wait() [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2109.015618] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] current.throw(*self._exc) [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] result = function(*args, **kwargs) [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] return func(*args, **kwargs) [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] raise e [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] nwinfo = self.network_api.allocate_for_instance( [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] created_port_ids = self._update_ports_for_instance( [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] with excutils.save_and_reraise_exception(): [ 2109.016012] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self.force_reraise() [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] raise self.value [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] updated_port = self._update_port( [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] _ensure_no_port_binding_failure(port) [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] raise exception.PortBindingFailed(port_id=port['id']) [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2109.016389] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] [ 2109.016389] env[62506]: INFO nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Terminating instance [ 2109.096866] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2109.249385] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.312s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2109.249844] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2109.252664] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.095s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2109.254347] env[62506]: INFO nova.compute.claims [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2109.330080] env[62506]: DEBUG nova.network.neutron [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2109.346504] env[62506]: DEBUG nova.compute.manager [req-403b9b95-1b44-484d-9b9e-5844319f93d6 req-c79c8b39-a386-4f8d-b5ba-c1ac41e8e374 service nova] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Received event network-vif-deleted-2b476cfb-59b3-4836-af68-f470d7b7c994 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2109.448276] env[62506]: DEBUG nova.network.neutron [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2109.520179] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2109.599447] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Releasing lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2109.600404] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2109.602640] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2109.602640] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fb651ed3-781b-4164-a025-330d96b694f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2109.612021] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a9db3a2-8605-40e8-93e3-b06e924fe5a6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2109.623574] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2109.635690] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance caa77bc8-f831-418e-bb46-bd7d5ed6ef2f could not be found. [ 2109.635903] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2109.636135] env[62506]: INFO nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2109.636335] env[62506]: DEBUG oslo.service.loopingcall [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2109.636555] env[62506]: DEBUG nova.compute.manager [-] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2109.636650] env[62506]: DEBUG nova.network.neutron [-] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2109.654046] env[62506]: DEBUG nova.network.neutron [-] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2109.759401] env[62506]: DEBUG nova.compute.utils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2109.763660] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2109.763865] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2109.825917] env[62506]: DEBUG nova.policy [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e508d2263c0c47d5bccfb1e2f1409316', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7d771ac74ac54fb38a84337eed72348a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2109.951025] env[62506]: DEBUG oslo_concurrency.lockutils [req-db38385a-720a-402b-a77a-695f2d017c9b req-2b127102-0a68-4c29-81de-fc0ffd8e7d92 service nova] Releasing lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2109.951551] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2109.951837] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2110.158721] env[62506]: DEBUG nova.network.neutron [-] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2110.264274] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2110.380759] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Successfully created port: 66313fe3-61a1-43de-8f38-f15a97daf222 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2110.470805] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2110.554220] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2110.590910] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68cdd088-7387-4e57-9193-f0bd21744fea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2110.598034] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11a76a29-85cd-43cd-a520-20fa7d7f55e8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2110.631771] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2110.636779] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abe5aa48-e314-4c30-b867-48fe3b526ef9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2110.646709] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cd5dab2-2318-47a1-a9da-c8c59c196d7b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2110.661412] env[62506]: INFO nova.compute.manager [-] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Took 1.02 seconds to deallocate network for instance. [ 2110.661923] env[62506]: DEBUG nova.compute.provider_tree [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2110.665935] env[62506]: DEBUG nova.compute.claims [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2110.666036] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2110.831690] env[62506]: DEBUG nova.compute.manager [req-4beca3f7-3b4b-4256-b217-0d29fa247d56 req-fd5fbc28-b051-4974-94a6-095ba003ced1 service nova] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Received event network-vif-deleted-15763fc5-3cde-4c99-84fe-bc6a7747d33a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2111.057809] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2111.058244] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2111.058442] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2111.058756] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-586c5fac-4f10-4c57-b975-6d07800de50e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2111.069342] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d508fb4d-19b0-4643-92d8-ca419fc7eb96 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2111.093332] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6d89deb7-cc37-4feb-a998-fef1d8a92947 could not be found. [ 2111.093332] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2111.093332] env[62506]: INFO nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2111.093332] env[62506]: DEBUG oslo.service.loopingcall [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2111.093332] env[62506]: DEBUG nova.compute.manager [-] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2111.093332] env[62506]: DEBUG nova.network.neutron [-] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2111.111828] env[62506]: DEBUG nova.network.neutron [-] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2111.143419] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2111.166780] env[62506]: DEBUG nova.scheduler.client.report [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2111.263025] env[62506]: ERROR nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2111.263025] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2111.263025] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2111.263025] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2111.263025] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2111.263025] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2111.263025] env[62506]: ERROR nova.compute.manager raise self.value [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2111.263025] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2111.263025] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2111.263025] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2111.263510] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2111.263510] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2111.263510] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2111.263510] env[62506]: ERROR nova.compute.manager [ 2111.263510] env[62506]: Traceback (most recent call last): [ 2111.263510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2111.263510] env[62506]: listener.cb(fileno) [ 2111.263510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2111.263510] env[62506]: result = function(*args, **kwargs) [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2111.263510] env[62506]: return func(*args, **kwargs) [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2111.263510] env[62506]: raise e [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2111.263510] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2111.263510] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2111.263510] env[62506]: with excutils.save_and_reraise_exception(): [ 2111.263510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2111.263510] env[62506]: self.force_reraise() [ 2111.263510] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2111.263510] env[62506]: raise self.value [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2111.263510] env[62506]: updated_port = self._update_port( [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2111.263510] env[62506]: _ensure_no_port_binding_failure(port) [ 2111.263510] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2111.263510] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2111.264298] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2111.264298] env[62506]: Removing descriptor: 20 [ 2111.276289] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2111.308111] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2111.308111] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2111.308111] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2111.308111] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2111.308761] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2111.309071] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2111.309414] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2111.309723] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2111.310068] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2111.310431] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2111.310770] env[62506]: DEBUG nova.virt.hardware [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2111.311843] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11c98280-efbb-4567-a80c-916a749109e3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2111.321248] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e443883-db62-4119-8c35-a8e5776cd0bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2111.338458] env[62506]: ERROR nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Traceback (most recent call last): [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] yield resources [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self.driver.spawn(context, instance, image_meta, [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] vm_ref = self.build_virtual_machine(instance, [ 2111.338458] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] for vif in network_info: [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] return self._sync_wrapper(fn, *args, **kwargs) [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self.wait() [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self[:] = self._gt.wait() [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] return self._exit_event.wait() [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2111.338795] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] current.throw(*self._exc) [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] result = function(*args, **kwargs) [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] return func(*args, **kwargs) [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] raise e [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] nwinfo = self.network_api.allocate_for_instance( [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] created_port_ids = self._update_ports_for_instance( [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] with excutils.save_and_reraise_exception(): [ 2111.339114] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self.force_reraise() [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] raise self.value [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] updated_port = self._update_port( [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] _ensure_no_port_binding_failure(port) [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] raise exception.PortBindingFailed(port_id=port['id']) [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2111.339415] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] [ 2111.341074] env[62506]: INFO nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Terminating instance [ 2111.377782] env[62506]: DEBUG nova.compute.manager [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Received event network-changed-66313fe3-61a1-43de-8f38-f15a97daf222 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2111.378341] env[62506]: DEBUG nova.compute.manager [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Refreshing instance network info cache due to event network-changed-66313fe3-61a1-43de-8f38-f15a97daf222. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2111.378341] env[62506]: DEBUG oslo_concurrency.lockutils [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] Acquiring lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2111.378341] env[62506]: DEBUG oslo_concurrency.lockutils [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] Acquired lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2111.378592] env[62506]: DEBUG nova.network.neutron [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Refreshing network info cache for port 66313fe3-61a1-43de-8f38-f15a97daf222 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2111.614978] env[62506]: DEBUG nova.network.neutron [-] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2111.672113] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.419s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2111.674991] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2111.675978] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 16.907s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2111.676715] env[62506]: DEBUG nova.objects.instance [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lazy-loading 'resources' on Instance uuid 5e16bc57-424c-45b0-8e85-1d80bac92b18 {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 2111.845288] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2111.902711] env[62506]: DEBUG nova.network.neutron [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2111.987366] env[62506]: DEBUG nova.network.neutron [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2112.119449] env[62506]: INFO nova.compute.manager [-] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Took 1.03 seconds to deallocate network for instance. [ 2112.122081] env[62506]: DEBUG nova.compute.claims [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2112.122263] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2112.185597] env[62506]: DEBUG nova.compute.utils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2112.185597] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2112.185597] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2112.230864] env[62506]: DEBUG nova.policy [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '59f4559064a14dea9d74a4fef4eff6d7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '612309ae575f46bdb31861d769d09fd8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2112.394196] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-781eb423-7d3c-466d-b2f4-b966dc32cfc3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2112.402119] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f9390ff-7dfc-4702-b499-260342780175 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2112.435944] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-150ffd88-c7a7-4156-8585-900711c71fee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2112.443536] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a19f96a-a73a-4174-8937-904ac93e6cdc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2112.457755] env[62506]: DEBUG nova.compute.provider_tree [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2112.492114] env[62506]: DEBUG oslo_concurrency.lockutils [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] Releasing lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2112.492312] env[62506]: DEBUG nova.compute.manager [req-0de7b15f-5cb2-4934-bf8e-69c1357d2da8 req-f6b0d0fb-3a18-45cd-b950-b09f9dca6b72 service nova] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Received event network-vif-deleted-66313fe3-61a1-43de-8f38-f15a97daf222 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2112.492735] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2112.492906] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2112.541834] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Successfully created port: 38127312-6553-4ab8-8e8a-ebf3fe171bf7 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2112.694738] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2112.961479] env[62506]: DEBUG nova.scheduler.client.report [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2113.011133] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2113.104229] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2113.359679] env[62506]: ERROR nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2113.359679] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2113.359679] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2113.359679] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2113.359679] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2113.359679] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2113.359679] env[62506]: ERROR nova.compute.manager raise self.value [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2113.359679] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2113.359679] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2113.359679] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2113.360189] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2113.360189] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2113.360189] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2113.360189] env[62506]: ERROR nova.compute.manager [ 2113.360189] env[62506]: Traceback (most recent call last): [ 2113.360189] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2113.360189] env[62506]: listener.cb(fileno) [ 2113.360189] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2113.360189] env[62506]: result = function(*args, **kwargs) [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2113.360189] env[62506]: return func(*args, **kwargs) [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2113.360189] env[62506]: raise e [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2113.360189] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2113.360189] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2113.360189] env[62506]: with excutils.save_and_reraise_exception(): [ 2113.360189] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2113.360189] env[62506]: self.force_reraise() [ 2113.360189] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2113.360189] env[62506]: raise self.value [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2113.360189] env[62506]: updated_port = self._update_port( [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2113.360189] env[62506]: _ensure_no_port_binding_failure(port) [ 2113.360189] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2113.360189] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2113.360966] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2113.360966] env[62506]: Removing descriptor: 20 [ 2113.401627] env[62506]: DEBUG nova.compute.manager [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Received event network-changed-38127312-6553-4ab8-8e8a-ebf3fe171bf7 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2113.401803] env[62506]: DEBUG nova.compute.manager [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Refreshing instance network info cache due to event network-changed-38127312-6553-4ab8-8e8a-ebf3fe171bf7. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2113.402068] env[62506]: DEBUG oslo_concurrency.lockutils [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] Acquiring lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2113.402169] env[62506]: DEBUG oslo_concurrency.lockutils [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] Acquired lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2113.402331] env[62506]: DEBUG nova.network.neutron [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Refreshing network info cache for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2113.470764] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.795s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2113.474386] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.130s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2113.491636] env[62506]: INFO nova.scheduler.client.report [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Deleted allocations for instance 5e16bc57-424c-45b0-8e85-1d80bac92b18 [ 2113.606673] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2113.607134] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2113.607331] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2113.607644] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0c9257be-3f55-41cb-b2d1-122fd84940c4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2113.616892] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5896d0b-9697-417f-95a9-ddeca85979f1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2113.639356] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 91925f54-176b-4f34-8634-46bf7cb73bdd could not be found. [ 2113.639729] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2113.640070] env[62506]: INFO nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2113.640362] env[62506]: DEBUG oslo.service.loopingcall [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2113.640611] env[62506]: DEBUG nova.compute.manager [-] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2113.640700] env[62506]: DEBUG nova.network.neutron [-] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2113.655975] env[62506]: DEBUG nova.network.neutron [-] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2113.701830] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2113.727610] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2113.727947] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2113.728173] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2113.728411] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2113.728587] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2113.728768] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2113.729064] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2113.729286] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2113.729505] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2113.729703] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2113.729904] env[62506]: DEBUG nova.virt.hardware [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2113.731203] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f0e7898-a0e3-4b67-a564-930c2900001f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2113.739580] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f475a1b-c090-4403-997c-99385a807eb7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2113.753664] env[62506]: ERROR nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Traceback (most recent call last): [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] yield resources [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self.driver.spawn(context, instance, image_meta, [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] vm_ref = self.build_virtual_machine(instance, [ 2113.753664] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] vif_infos = vmwarevif.get_vif_info(self._session, [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] for vif in network_info: [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] return self._sync_wrapper(fn, *args, **kwargs) [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self.wait() [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self[:] = self._gt.wait() [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] return self._exit_event.wait() [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2113.754048] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] current.throw(*self._exc) [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] result = function(*args, **kwargs) [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] return func(*args, **kwargs) [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] raise e [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] nwinfo = self.network_api.allocate_for_instance( [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] created_port_ids = self._update_ports_for_instance( [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] with excutils.save_and_reraise_exception(): [ 2113.754393] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self.force_reraise() [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] raise self.value [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] updated_port = self._update_port( [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] _ensure_no_port_binding_failure(port) [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] raise exception.PortBindingFailed(port_id=port['id']) [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2113.754749] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] [ 2113.754749] env[62506]: INFO nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Terminating instance [ 2113.924437] env[62506]: DEBUG nova.network.neutron [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2114.008042] env[62506]: DEBUG oslo_concurrency.lockutils [None req-c29ffd5d-5de5-43e8-ac11-54e93ff339e5 tempest-ServerShowV247Test-910873107 tempest-ServerShowV247Test-910873107-project-member] Lock "5e16bc57-424c-45b0-8e85-1d80bac92b18" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 24.753s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2114.049770] env[62506]: DEBUG nova.network.neutron [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2114.158244] env[62506]: DEBUG nova.network.neutron [-] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2114.170193] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1669ebb-d716-4fa6-9766-1ba56707cd49 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2114.178026] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b1af536-2934-46cd-919d-2479031a49b0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2114.209214] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93cdf65a-3fd7-4b75-90a4-2203bffbfad1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2114.216436] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f18e2b09-d12c-4fcb-8de7-68ac95b8809f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2114.229577] env[62506]: DEBUG nova.compute.provider_tree [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2114.257772] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2114.553031] env[62506]: DEBUG oslo_concurrency.lockutils [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] Releasing lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2114.553031] env[62506]: DEBUG nova.compute.manager [req-ed659fa7-965a-439e-8074-6e7fed7500d9 req-89340806-4680-4e9c-a3ca-aaa491f8070f service nova] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Received event network-vif-deleted-38127312-6553-4ab8-8e8a-ebf3fe171bf7 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2114.553363] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquired lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2114.553561] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2114.661799] env[62506]: INFO nova.compute.manager [-] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Took 1.02 seconds to deallocate network for instance. [ 2114.664509] env[62506]: DEBUG nova.compute.claims [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2114.664686] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2114.733280] env[62506]: DEBUG nova.scheduler.client.report [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2115.076323] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2115.168197] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2115.242941] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.769s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2115.244169] env[62506]: ERROR nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Traceback (most recent call last): [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self.driver.spawn(context, instance, image_meta, [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] vm_ref = self.build_virtual_machine(instance, [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] vif_infos = vmwarevif.get_vif_info(self._session, [ 2115.244169] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] for vif in network_info: [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return self._sync_wrapper(fn, *args, **kwargs) [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self.wait() [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self[:] = self._gt.wait() [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return self._exit_event.wait() [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] result = hub.switch() [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2115.244649] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return self.greenlet.switch() [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] result = function(*args, **kwargs) [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] return func(*args, **kwargs) [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] raise e [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] nwinfo = self.network_api.allocate_for_instance( [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] created_port_ids = self._update_ports_for_instance( [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] with excutils.save_and_reraise_exception(): [ 2115.245050] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] self.force_reraise() [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] raise self.value [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] updated_port = self._update_port( [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] _ensure_no_port_binding_failure(port) [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] raise exception.PortBindingFailed(port_id=port['id']) [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] nova.exception.PortBindingFailed: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. [ 2115.245453] env[62506]: ERROR nova.compute.manager [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] [ 2115.245812] env[62506]: DEBUG nova.compute.utils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2115.248234] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Build of instance b45f2334-c604-4454-993e-e1c16a6c1ef5 was re-scheduled: Binding failed for port 68eed6aa-a43d-4488-96ec-b81369cc171a, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2115.248994] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2115.249423] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquiring lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2115.249715] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Acquired lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2115.250047] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2115.251458] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.029s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2115.253309] env[62506]: INFO nova.compute.claims [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2115.671199] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Releasing lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2115.671647] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2115.671846] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2115.672183] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-febd786c-baea-4ea5-b31e-cb3b17cd08a5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2115.682618] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47dc3c64-44bc-49ff-b954-3bcf5f4f1d60 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2115.704445] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 80c76c85-2575-4733-a86a-111d138d69d4 could not be found. [ 2115.704692] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2115.704856] env[62506]: INFO nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2115.705149] env[62506]: DEBUG oslo.service.loopingcall [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2115.705386] env[62506]: DEBUG nova.compute.manager [-] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2115.705483] env[62506]: DEBUG nova.network.neutron [-] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2115.721898] env[62506]: DEBUG nova.network.neutron [-] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2115.777974] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2115.865408] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2116.226807] env[62506]: DEBUG nova.network.neutron [-] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2116.367810] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Releasing lock "refresh_cache-b45f2334-c604-4454-993e-e1c16a6c1ef5" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2116.368050] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2116.370832] env[62506]: DEBUG nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2116.370832] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2116.387024] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2116.448327] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0aa6ae61-43c1-4bc6-a1d3-3c51a429653b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.455922] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-487d490a-8673-4e2b-b9a7-f56b4d8eaa51 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.490047] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebfa94e3-d07f-4fa9-a303-242fc456b496 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.497708] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c98345dc-f0e0-4c1c-8e43-6fb66f05f3a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.511203] env[62506]: DEBUG nova.compute.provider_tree [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2116.729771] env[62506]: INFO nova.compute.manager [-] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Took 1.02 seconds to deallocate network for instance. [ 2116.732349] env[62506]: DEBUG nova.compute.claims [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2116.732622] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2116.890982] env[62506]: DEBUG nova.network.neutron [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2117.046967] env[62506]: DEBUG nova.scheduler.client.report [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 89 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 2117.047268] env[62506]: DEBUG nova.compute.provider_tree [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 89 to 90 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 2117.047455] env[62506]: DEBUG nova.compute.provider_tree [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2117.397439] env[62506]: INFO nova.compute.manager [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] [instance: b45f2334-c604-4454-993e-e1c16a6c1ef5] Took 1.03 seconds to deallocate network for instance. [ 2117.552926] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.301s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2117.553506] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2117.557201] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.704s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2117.559514] env[62506]: INFO nova.compute.claims [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2118.066115] env[62506]: DEBUG nova.compute.utils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2118.073321] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2118.073321] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2118.129761] env[62506]: DEBUG nova.policy [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a01d435093084caaa85feba239fcb70b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5c5301143ee049a9b5ccff80620d12ae', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2118.398531] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Successfully created port: eb89e77d-ab13-4eb6-973b-a04d23eef0db {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2118.430285] env[62506]: INFO nova.scheduler.client.report [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Deleted allocations for instance b45f2334-c604-4454-993e-e1c16a6c1ef5 [ 2118.576469] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2118.697519] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "220ccae9-6d65-4925-9fef-d555cb923fdd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2118.697756] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "220ccae9-6d65-4925-9fef-d555cb923fdd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2118.749416] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9b4ed22-77a2-4584-896a-50722dfe4ca9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2118.756940] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5af2f030-07b1-4182-bfba-741479eff7be {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2118.789226] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e25cfdf-ae3d-44db-89d5-9ae656ef17b7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2118.796557] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9689f587-f2d4-497e-9d9f-c51be3a1a0e3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2118.809709] env[62506]: DEBUG nova.compute.provider_tree [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2118.943155] env[62506]: DEBUG oslo_concurrency.lockutils [None req-7c7ee1b2-9b57-4c70-8349-0ecfcfd375a0 tempest-ListServerFiltersTestJSON-1856073696 tempest-ListServerFiltersTestJSON-1856073696-project-member] Lock "b45f2334-c604-4454-993e-e1c16a6c1ef5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 103.917s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2119.125482] env[62506]: DEBUG nova.compute.manager [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Received event network-changed-eb89e77d-ab13-4eb6-973b-a04d23eef0db {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2119.125717] env[62506]: DEBUG nova.compute.manager [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Refreshing instance network info cache due to event network-changed-eb89e77d-ab13-4eb6-973b-a04d23eef0db. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2119.125954] env[62506]: DEBUG oslo_concurrency.lockutils [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] Acquiring lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2119.126365] env[62506]: DEBUG oslo_concurrency.lockutils [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] Acquired lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2119.126365] env[62506]: DEBUG nova.network.neutron [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Refreshing network info cache for port eb89e77d-ab13-4eb6-973b-a04d23eef0db {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2119.289878] env[62506]: ERROR nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2119.289878] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2119.289878] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2119.289878] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2119.289878] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2119.289878] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2119.289878] env[62506]: ERROR nova.compute.manager raise self.value [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2119.289878] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2119.289878] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2119.289878] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2119.290370] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2119.290370] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2119.290370] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2119.290370] env[62506]: ERROR nova.compute.manager [ 2119.290370] env[62506]: Traceback (most recent call last): [ 2119.290370] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2119.290370] env[62506]: listener.cb(fileno) [ 2119.290370] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2119.290370] env[62506]: result = function(*args, **kwargs) [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2119.290370] env[62506]: return func(*args, **kwargs) [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2119.290370] env[62506]: raise e [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2119.290370] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2119.290370] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2119.290370] env[62506]: with excutils.save_and_reraise_exception(): [ 2119.290370] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2119.290370] env[62506]: self.force_reraise() [ 2119.290370] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2119.290370] env[62506]: raise self.value [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2119.290370] env[62506]: updated_port = self._update_port( [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2119.290370] env[62506]: _ensure_no_port_binding_failure(port) [ 2119.290370] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2119.290370] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2119.291278] env[62506]: nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2119.291278] env[62506]: Removing descriptor: 20 [ 2119.313323] env[62506]: DEBUG nova.scheduler.client.report [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2119.446276] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2119.589287] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2119.615262] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2119.615524] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2119.615685] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2119.615867] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2119.616022] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2119.616174] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2119.616382] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2119.616538] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2119.616724] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2119.616863] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2119.617045] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2119.617931] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5673ea97-da86-4a3b-932f-28a36426b47f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2119.626356] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81e16e51-794f-43eb-aad2-1743d38ce284 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2119.644032] env[62506]: ERROR nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Traceback (most recent call last): [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] yield resources [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self.driver.spawn(context, instance, image_meta, [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] vm_ref = self.build_virtual_machine(instance, [ 2119.644032] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] vif_infos = vmwarevif.get_vif_info(self._session, [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] for vif in network_info: [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] return self._sync_wrapper(fn, *args, **kwargs) [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self.wait() [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self[:] = self._gt.wait() [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] return self._exit_event.wait() [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2119.644438] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] current.throw(*self._exc) [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] result = function(*args, **kwargs) [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] return func(*args, **kwargs) [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] raise e [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] nwinfo = self.network_api.allocate_for_instance( [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] created_port_ids = self._update_ports_for_instance( [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] with excutils.save_and_reraise_exception(): [ 2119.644801] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self.force_reraise() [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] raise self.value [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] updated_port = self._update_port( [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] _ensure_no_port_binding_failure(port) [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] raise exception.PortBindingFailed(port_id=port['id']) [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2119.645227] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] [ 2119.645227] env[62506]: INFO nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Terminating instance [ 2119.647693] env[62506]: DEBUG nova.network.neutron [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2119.719713] env[62506]: DEBUG nova.network.neutron [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2119.818726] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.261s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2119.819287] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2119.823049] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.514s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2119.970316] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2120.151315] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2120.222612] env[62506]: DEBUG oslo_concurrency.lockutils [req-52056a23-9b8a-4e79-ac63-8fd84423d095 req-3b4b2848-d2b2-4cba-8258-ecc55a7dce22 service nova] Releasing lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2120.223067] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2120.223264] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2120.328651] env[62506]: DEBUG nova.compute.utils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2120.333568] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2120.333618] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2120.379615] env[62506]: DEBUG nova.policy [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a01d435093084caaa85feba239fcb70b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5c5301143ee049a9b5ccff80620d12ae', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2120.508682] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f6555e4-6a11-404c-be24-9c064efa71c4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2120.516871] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3da87b3c-7033-462e-925b-d3af7fcfc1c6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2120.546583] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd8bc4ea-d872-4925-afe4-363ddb61ac5f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2120.553873] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20fd5a53-dab4-4c03-a969-c368c84b6d24 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2120.566769] env[62506]: DEBUG nova.compute.provider_tree [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2120.645042] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Successfully created port: ec1d4c39-466a-49b1-a469-49fd06de3148 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2120.745041] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2120.839271] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2120.857464] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2121.073179] env[62506]: DEBUG nova.scheduler.client.report [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2121.152298] env[62506]: DEBUG nova.compute.manager [req-f8d38297-d1e4-4f87-806e-a8454593ac4d req-ff0166e2-dcd8-44aa-a7fe-a6b88de08974 service nova] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Received event network-vif-deleted-eb89e77d-ab13-4eb6-973b-a04d23eef0db {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2121.361334] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2121.361760] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2121.361964] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2121.362310] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2cdfaf56-ce9c-4a77-b65e-92299dec4adf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2121.372334] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-095fc8c9-32a5-4b63-891f-d4efdf0a4b8b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2121.397291] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1147abf9-da6d-4e51-878e-b8dc9a3f6141 could not be found. [ 2121.397533] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2121.397722] env[62506]: INFO nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2121.397975] env[62506]: DEBUG oslo.service.loopingcall [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2121.398248] env[62506]: DEBUG nova.compute.manager [-] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2121.398346] env[62506]: DEBUG nova.network.neutron [-] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2121.414552] env[62506]: DEBUG nova.network.neutron [-] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2121.494431] env[62506]: ERROR nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2121.494431] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2121.494431] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2121.494431] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2121.494431] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2121.494431] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2121.494431] env[62506]: ERROR nova.compute.manager raise self.value [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2121.494431] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2121.494431] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2121.494431] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2121.495084] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2121.495084] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2121.495084] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2121.495084] env[62506]: ERROR nova.compute.manager [ 2121.495084] env[62506]: Traceback (most recent call last): [ 2121.495084] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2121.495084] env[62506]: listener.cb(fileno) [ 2121.495084] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2121.495084] env[62506]: result = function(*args, **kwargs) [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2121.495084] env[62506]: return func(*args, **kwargs) [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2121.495084] env[62506]: raise e [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2121.495084] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2121.495084] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2121.495084] env[62506]: with excutils.save_and_reraise_exception(): [ 2121.495084] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2121.495084] env[62506]: self.force_reraise() [ 2121.495084] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2121.495084] env[62506]: raise self.value [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2121.495084] env[62506]: updated_port = self._update_port( [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2121.495084] env[62506]: _ensure_no_port_binding_failure(port) [ 2121.495084] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2121.495084] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2121.495969] env[62506]: nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2121.495969] env[62506]: Removing descriptor: 20 [ 2121.578399] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.756s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2121.578991] env[62506]: ERROR nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Traceback (most recent call last): [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self.driver.spawn(context, instance, image_meta, [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] vm_ref = self.build_virtual_machine(instance, [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] vif_infos = vmwarevif.get_vif_info(self._session, [ 2121.578991] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] for vif in network_info: [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] return self._sync_wrapper(fn, *args, **kwargs) [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self.wait() [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self[:] = self._gt.wait() [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] return self._exit_event.wait() [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] current.throw(*self._exc) [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2121.579406] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] result = function(*args, **kwargs) [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] return func(*args, **kwargs) [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] raise e [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] nwinfo = self.network_api.allocate_for_instance( [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] created_port_ids = self._update_ports_for_instance( [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] with excutils.save_and_reraise_exception(): [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] self.force_reraise() [ 2121.579808] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] raise self.value [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] updated_port = self._update_port( [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] _ensure_no_port_binding_failure(port) [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] raise exception.PortBindingFailed(port_id=port['id']) [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] nova.exception.PortBindingFailed: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. [ 2121.580164] env[62506]: ERROR nova.compute.manager [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] [ 2121.580164] env[62506]: DEBUG nova.compute.utils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2121.581016] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.154s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2121.584103] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Build of instance 4ec5b60e-a35c-40b5-bf8b-167b9d856f98 was re-scheduled: Binding failed for port 1b0733dc-bfc2-427d-b62f-aed00409289f, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2121.584576] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2121.584807] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2121.584954] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2121.585129] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2121.848749] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2121.876442] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2121.876699] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2121.876855] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2121.877048] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2121.877200] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2121.877350] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2121.877554] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2121.877710] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2121.877876] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2121.878057] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2121.878233] env[62506]: DEBUG nova.virt.hardware [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2121.879401] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4b6e751-7df6-43e7-909d-98ced75f77c2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2121.888712] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f46bd0d7-5fa5-49dc-a421-5e9386ed58d7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2121.902497] env[62506]: ERROR nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Traceback (most recent call last): [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] yield resources [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self.driver.spawn(context, instance, image_meta, [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] vm_ref = self.build_virtual_machine(instance, [ 2121.902497] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] vif_infos = vmwarevif.get_vif_info(self._session, [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] for vif in network_info: [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] return self._sync_wrapper(fn, *args, **kwargs) [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self.wait() [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self[:] = self._gt.wait() [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] return self._exit_event.wait() [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2121.902905] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] current.throw(*self._exc) [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] result = function(*args, **kwargs) [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] return func(*args, **kwargs) [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] raise e [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] nwinfo = self.network_api.allocate_for_instance( [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] created_port_ids = self._update_ports_for_instance( [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] with excutils.save_and_reraise_exception(): [ 2121.903243] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self.force_reraise() [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] raise self.value [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] updated_port = self._update_port( [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] _ensure_no_port_binding_failure(port) [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] raise exception.PortBindingFailed(port_id=port['id']) [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2121.903655] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] [ 2121.903655] env[62506]: INFO nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Terminating instance [ 2121.916445] env[62506]: DEBUG nova.network.neutron [-] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2122.107550] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2122.190864] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2122.236090] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2551e14c-404e-4d91-9520-4316343a5ad1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2122.242815] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dead5d3-a67b-4bf0-9143-e9cd61bcc67b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2122.274031] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3abc4d4-7980-4a96-9644-e7013187bb4f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2122.281063] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-953c2fda-5915-42f7-a1bc-b87455d234d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2122.293036] env[62506]: DEBUG nova.compute.provider_tree [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2122.407071] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2122.407269] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2122.407445] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2122.418593] env[62506]: INFO nova.compute.manager [-] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Took 1.02 seconds to deallocate network for instance. [ 2122.420663] env[62506]: DEBUG nova.compute.claims [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2122.420831] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2122.693243] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-4ec5b60e-a35c-40b5-bf8b-167b9d856f98" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2122.693501] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2122.693706] env[62506]: DEBUG nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2122.693882] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2122.710848] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2122.796143] env[62506]: DEBUG nova.scheduler.client.report [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2122.924637] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2122.995368] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2123.177303] env[62506]: DEBUG nova.compute.manager [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Received event network-changed-ec1d4c39-466a-49b1-a469-49fd06de3148 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2123.177669] env[62506]: DEBUG nova.compute.manager [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Refreshing instance network info cache due to event network-changed-ec1d4c39-466a-49b1-a469-49fd06de3148. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2123.177919] env[62506]: DEBUG oslo_concurrency.lockutils [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] Acquiring lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2123.213438] env[62506]: DEBUG nova.network.neutron [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2123.300702] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.720s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2123.301421] env[62506]: ERROR nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] Traceback (most recent call last): [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self.driver.spawn(context, instance, image_meta, [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] vm_ref = self.build_virtual_machine(instance, [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] vif_infos = vmwarevif.get_vif_info(self._session, [ 2123.301421] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] for vif in network_info: [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] return self._sync_wrapper(fn, *args, **kwargs) [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self.wait() [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self[:] = self._gt.wait() [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] return self._exit_event.wait() [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] current.throw(*self._exc) [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2123.301751] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] result = function(*args, **kwargs) [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] return func(*args, **kwargs) [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] raise e [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] nwinfo = self.network_api.allocate_for_instance( [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] created_port_ids = self._update_ports_for_instance( [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] with excutils.save_and_reraise_exception(): [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] self.force_reraise() [ 2123.302080] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] raise self.value [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] updated_port = self._update_port( [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] _ensure_no_port_binding_failure(port) [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] raise exception.PortBindingFailed(port_id=port['id']) [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] nova.exception.PortBindingFailed: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. [ 2123.302426] env[62506]: ERROR nova.compute.manager [instance: 94a23198-2d64-4003-93cc-93522fe70172] [ 2123.302426] env[62506]: DEBUG nova.compute.utils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2123.303392] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.161s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2123.304818] env[62506]: INFO nova.compute.claims [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2123.308794] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Build of instance 94a23198-2d64-4003-93cc-93522fe70172 was re-scheduled: Binding failed for port fe682248-e43e-417d-b795-828cfafe5d2e, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2123.309240] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2123.309465] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquiring lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2123.309612] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Acquired lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2123.309773] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2123.498171] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2123.498552] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2123.498748] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2123.499072] env[62506]: DEBUG oslo_concurrency.lockutils [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] Acquired lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2123.499253] env[62506]: DEBUG nova.network.neutron [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Refreshing network info cache for port ec1d4c39-466a-49b1-a469-49fd06de3148 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2123.501703] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6d9c9c2f-2629-4e1c-b07d-afb2acf0035d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2123.511313] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d78c946-70ff-4538-b511-19a37ef98667 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2123.533031] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d4e49d27-403b-4f85-95b7-ae07e9bb07b8 could not be found. [ 2123.533250] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2123.533430] env[62506]: INFO nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2123.533706] env[62506]: DEBUG oslo.service.loopingcall [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2123.533932] env[62506]: DEBUG nova.compute.manager [-] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2123.534038] env[62506]: DEBUG nova.network.neutron [-] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2123.549016] env[62506]: DEBUG nova.network.neutron [-] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2123.716460] env[62506]: INFO nova.compute.manager [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 4ec5b60e-a35c-40b5-bf8b-167b9d856f98] Took 1.02 seconds to deallocate network for instance. [ 2123.828020] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2123.902612] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2124.020299] env[62506]: DEBUG nova.network.neutron [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2124.051422] env[62506]: DEBUG nova.network.neutron [-] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2124.092369] env[62506]: DEBUG nova.network.neutron [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2124.406436] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Releasing lock "refresh_cache-94a23198-2d64-4003-93cc-93522fe70172" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2124.406541] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2124.406735] env[62506]: DEBUG nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2124.406846] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2124.421744] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2124.454122] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6695e598-656d-4471-af4e-2ac66d132726 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.462097] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e902ae31-3ed3-4feb-9b52-9583958e2818 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.493418] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b18d5df3-9dff-4f40-8f08-c948cc7f253e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.500970] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-042f76a7-9239-4108-a1f8-fa053e2ccb84 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.514113] env[62506]: DEBUG nova.compute.provider_tree [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2124.554251] env[62506]: INFO nova.compute.manager [-] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Took 1.02 seconds to deallocate network for instance. [ 2124.556498] env[62506]: DEBUG nova.compute.claims [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2124.556671] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2124.595808] env[62506]: DEBUG oslo_concurrency.lockutils [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] Releasing lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2124.596073] env[62506]: DEBUG nova.compute.manager [req-8288e576-3ca2-41cd-888e-6ce81ebe0df0 req-a706019b-4659-4770-bb7e-2b5b1db556c7 service nova] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Received event network-vif-deleted-ec1d4c39-466a-49b1-a469-49fd06de3148 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2124.743562] env[62506]: INFO nova.scheduler.client.report [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Deleted allocations for instance 4ec5b60e-a35c-40b5-bf8b-167b9d856f98 [ 2124.924108] env[62506]: DEBUG nova.network.neutron [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2125.016942] env[62506]: DEBUG nova.scheduler.client.report [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2125.251313] env[62506]: DEBUG oslo_concurrency.lockutils [None req-6aa3c554-0ac1-4827-8209-f4cd1fb133d0 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "4ec5b60e-a35c-40b5-bf8b-167b9d856f98" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 104.198s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2125.427041] env[62506]: INFO nova.compute.manager [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] [instance: 94a23198-2d64-4003-93cc-93522fe70172] Took 1.02 seconds to deallocate network for instance. [ 2125.525446] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.222s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2125.526013] env[62506]: DEBUG nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2125.528501] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.862s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2125.754346] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2126.036056] env[62506]: DEBUG nova.compute.utils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2126.037857] env[62506]: DEBUG nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 2126.170045] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d893e78-1dbb-4bf4-9c17-52484aed9d73 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2126.178484] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c60424cf-f9cf-4e61-acac-8c1231464e37 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2126.212024] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7188f93-03f8-46a3-a8e9-e8638bd7be98 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2126.220946] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3f30433-7670-4cd8-9a76-5ad6d0c13d6d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2126.234978] env[62506]: DEBUG nova.compute.provider_tree [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2126.279445] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2126.456354] env[62506]: INFO nova.scheduler.client.report [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Deleted allocations for instance 94a23198-2d64-4003-93cc-93522fe70172 [ 2126.539654] env[62506]: DEBUG nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2126.738384] env[62506]: DEBUG nova.scheduler.client.report [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2126.812437] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "ed4f2610-e5dd-4c56-bcc0-71ec198f4833" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2126.812753] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "ed4f2610-e5dd-4c56-bcc0-71ec198f4833" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2126.965624] env[62506]: DEBUG oslo_concurrency.lockutils [None req-33d6f1d3-d886-4344-a548-32e72df7b0b2 tempest-ServersV294TestFqdnHostnames-2110423044 tempest-ServersV294TestFqdnHostnames-2110423044-project-member] Lock "94a23198-2d64-4003-93cc-93522fe70172" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 99.840s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2127.243654] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.715s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2127.247176] env[62506]: ERROR nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Traceback (most recent call last): [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self.driver.spawn(context, instance, image_meta, [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] vm_ref = self.build_virtual_machine(instance, [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] vif_infos = vmwarevif.get_vif_info(self._session, [ 2127.247176] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] for vif in network_info: [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] return self._sync_wrapper(fn, *args, **kwargs) [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self.wait() [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self[:] = self._gt.wait() [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] return self._exit_event.wait() [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] current.throw(*self._exc) [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2127.247550] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] result = function(*args, **kwargs) [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] return func(*args, **kwargs) [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] raise e [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] nwinfo = self.network_api.allocate_for_instance( [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] created_port_ids = self._update_ports_for_instance( [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] with excutils.save_and_reraise_exception(): [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] self.force_reraise() [ 2127.247976] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] raise self.value [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] updated_port = self._update_port( [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] _ensure_no_port_binding_failure(port) [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] raise exception.PortBindingFailed(port_id=port['id']) [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] nova.exception.PortBindingFailed: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. [ 2127.248307] env[62506]: ERROR nova.compute.manager [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] [ 2127.248307] env[62506]: DEBUG nova.compute.utils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2127.249674] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 16.106s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2127.249974] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2127.250214] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2127.250599] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.128s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2127.253677] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Build of instance caa77bc8-f831-418e-bb46-bd7d5ed6ef2f was re-scheduled: Binding failed for port 2b476cfb-59b3-4836-af68-f470d7b7c994, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2127.254165] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2127.254403] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquiring lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2127.254573] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Acquired lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2127.254746] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2127.256383] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d060f2e-8157-4020-8555-d1cde3dc2fac {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.265749] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adef2e27-7d12-44e2-b4eb-c9a4ba098caf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.280902] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b3806f7-5b53-48e6-8947-c767ce373ee6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.288495] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-544dadf7-5359-48d5-9580-aa8844a058c9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.318540] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181024MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2127.318710] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2127.468665] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2127.551106] env[62506]: DEBUG nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2127.577036] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2127.577324] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2127.577482] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2127.577695] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2127.577847] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2127.578021] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2127.578260] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2127.578486] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2127.578609] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2127.578776] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2127.578985] env[62506]: DEBUG nova.virt.hardware [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2127.580201] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c863068-9541-4cfe-8203-f3af29979a85 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.588525] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f501230-1638-4feb-a93b-bdcf1bb3c77f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.602641] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2127.608151] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Creating folder: Project (98180e6ec3d84498bf16b890ca51553f). Parent ref: group-v446953. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 2127.608429] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f66e952a-7220-43a5-8abb-ebdd6cc32553 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.619257] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Created folder: Project (98180e6ec3d84498bf16b890ca51553f) in parent group-v446953. [ 2127.619442] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Creating folder: Instances. Parent ref: group-v446980. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 2127.619659] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2f088a20-eb6c-486e-9bc8-b7a180880ebe {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.627880] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Created folder: Instances in parent group-v446980. [ 2127.628123] env[62506]: DEBUG oslo.service.loopingcall [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2127.628310] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2127.628498] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-34fa9434-a1b1-463e-8aa1-fa2d3cdf1b93 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.644807] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2127.644807] env[62506]: value = "task-2190877" [ 2127.644807] env[62506]: _type = "Task" [ 2127.644807] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2127.652533] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190877, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2127.784326] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2127.912582] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fb7a78d-207c-4db6-86ef-e9520c7ee288 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.920023] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b25fffe-3649-4cff-93b9-1a061a89a40e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.949252] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef678297-8aa1-4c7a-986f-8bc40124ff71 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.956675] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e1f4fe0-ca29-4470-9108-dde3470caa82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2127.969731] env[62506]: DEBUG nova.compute.provider_tree [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2127.986545] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2128.049238] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2128.156112] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190877, 'name': CreateVM_Task, 'duration_secs': 0.259346} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2128.156282] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2128.156713] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2128.156874] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2128.157423] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2128.157687] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-33af46c9-9b9b-4ba6-b6cb-7b46b1e95fda {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2128.162111] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2128.162111] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c4c8b5-a58b-402f-b021-eb401f6ff997" [ 2128.162111] env[62506]: _type = "Task" [ 2128.162111] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2128.169066] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52c4c8b5-a58b-402f-b021-eb401f6ff997, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2128.472808] env[62506]: DEBUG nova.scheduler.client.report [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2128.557018] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Releasing lock "refresh_cache-caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2128.557018] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2128.557018] env[62506]: DEBUG nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2128.557018] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2128.573271] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2128.672608] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52c4c8b5-a58b-402f-b021-eb401f6ff997, 'name': SearchDatastore_Task, 'duration_secs': 0.011773} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2128.673012] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2128.673313] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2128.673580] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2128.674111] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2128.674378] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2128.674663] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-39dece12-c34b-47fc-acd2-f02758aeef5c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2128.682542] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2128.682746] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2128.683451] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-abd5cebb-0a6e-43be-9aae-51f70fe5df47 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2128.688184] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2128.688184] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52497692-6e1c-f1a1-25ab-6155d095368b" [ 2128.688184] env[62506]: _type = "Task" [ 2128.688184] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2128.695343] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52497692-6e1c-f1a1-25ab-6155d095368b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2128.977251] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.727s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2128.977915] env[62506]: ERROR nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Traceback (most recent call last): [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self.driver.spawn(context, instance, image_meta, [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] vm_ref = self.build_virtual_machine(instance, [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] vif_infos = vmwarevif.get_vif_info(self._session, [ 2128.977915] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] for vif in network_info: [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] return self._sync_wrapper(fn, *args, **kwargs) [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self.wait() [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self[:] = self._gt.wait() [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] return self._exit_event.wait() [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] current.throw(*self._exc) [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2128.978297] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] result = function(*args, **kwargs) [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] return func(*args, **kwargs) [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] raise e [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] nwinfo = self.network_api.allocate_for_instance( [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] created_port_ids = self._update_ports_for_instance( [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] with excutils.save_and_reraise_exception(): [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] self.force_reraise() [ 2128.978657] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] raise self.value [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] updated_port = self._update_port( [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] _ensure_no_port_binding_failure(port) [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] raise exception.PortBindingFailed(port_id=port['id']) [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] nova.exception.PortBindingFailed: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. [ 2128.979047] env[62506]: ERROR nova.compute.manager [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] [ 2128.979047] env[62506]: DEBUG nova.compute.utils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2128.979816] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.315s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2128.982901] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Build of instance 6d89deb7-cc37-4feb-a998-fef1d8a92947 was re-scheduled: Binding failed for port 15763fc5-3cde-4c99-84fe-bc6a7747d33a, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2128.983517] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2128.983610] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquiring lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2128.983712] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Acquired lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2128.983889] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2129.075997] env[62506]: DEBUG nova.network.neutron [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2129.199096] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52497692-6e1c-f1a1-25ab-6155d095368b, 'name': SearchDatastore_Task, 'duration_secs': 0.007523} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2129.200060] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c9f3dd7a-9b22-49fe-9ef6-97108e8a097a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2129.205711] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2129.205711] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5273a54c-6903-de8b-0cf8-6baff93ac00a" [ 2129.205711] env[62506]: _type = "Task" [ 2129.205711] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2129.213538] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5273a54c-6903-de8b-0cf8-6baff93ac00a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2129.506355] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2129.577332] env[62506]: INFO nova.compute.manager [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] [instance: caa77bc8-f831-418e-bb46-bd7d5ed6ef2f] Took 1.02 seconds to deallocate network for instance. [ 2129.604666] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2129.661174] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66423bf9-e773-4752-b2f1-b606b84467ba {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2129.669672] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38a4b891-1f42-475a-9158-18a704f7f358 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2129.700726] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30b88b50-bfe5-4775-8c3d-558c1fe198ea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2129.712428] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22ed4de5-d654-4532-814b-13d4c9f9fbf4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2129.723908] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5273a54c-6903-de8b-0cf8-6baff93ac00a, 'name': SearchDatastore_Task, 'duration_secs': 0.009196} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2129.731903] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2129.732228] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2129.732773] env[62506]: DEBUG nova.compute.provider_tree [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2129.734069] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a5f84b56-0837-4516-876d-39da00869e66 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2129.741571] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2129.741571] env[62506]: value = "task-2190878" [ 2129.741571] env[62506]: _type = "Task" [ 2129.741571] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2129.750724] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190878, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2130.107792] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Releasing lock "refresh_cache-6d89deb7-cc37-4feb-a998-fef1d8a92947" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2130.108229] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2130.108332] env[62506]: DEBUG nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2130.108505] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2130.126287] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2130.238296] env[62506]: DEBUG nova.scheduler.client.report [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2130.252033] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190878, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.457987} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2130.252376] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2130.252765] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2130.252995] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-acec5128-7425-4296-b7db-c161c33ab967 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2130.260146] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2130.260146] env[62506]: value = "task-2190879" [ 2130.260146] env[62506]: _type = "Task" [ 2130.260146] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2130.267915] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190879, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2130.553154] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquiring lock "99973068-d1a5-4809-b722-b6dd24ad3a12" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2130.553498] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Lock "99973068-d1a5-4809-b722-b6dd24ad3a12" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2130.610023] env[62506]: INFO nova.scheduler.client.report [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Deleted allocations for instance caa77bc8-f831-418e-bb46-bd7d5ed6ef2f [ 2130.629185] env[62506]: DEBUG nova.network.neutron [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2130.743606] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.764s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2130.744359] env[62506]: ERROR nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Traceback (most recent call last): [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self.driver.spawn(context, instance, image_meta, [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] vm_ref = self.build_virtual_machine(instance, [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2130.744359] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] for vif in network_info: [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] return self._sync_wrapper(fn, *args, **kwargs) [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self.wait() [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self[:] = self._gt.wait() [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] return self._exit_event.wait() [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] current.throw(*self._exc) [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2130.744718] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] result = function(*args, **kwargs) [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] return func(*args, **kwargs) [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] raise e [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] nwinfo = self.network_api.allocate_for_instance( [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] created_port_ids = self._update_ports_for_instance( [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] with excutils.save_and_reraise_exception(): [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] self.force_reraise() [ 2130.745248] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] raise self.value [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] updated_port = self._update_port( [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] _ensure_no_port_binding_failure(port) [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] raise exception.PortBindingFailed(port_id=port['id']) [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] nova.exception.PortBindingFailed: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. [ 2130.745847] env[62506]: ERROR nova.compute.manager [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] [ 2130.745847] env[62506]: DEBUG nova.compute.utils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2130.746779] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Build of instance 91925f54-176b-4f34-8634-46bf7cb73bdd was re-scheduled: Binding failed for port 66313fe3-61a1-43de-8f38-f15a97daf222, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2130.747236] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2130.747468] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2130.747617] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2130.747810] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2130.749204] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.017s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2130.769446] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190879, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.060034} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2130.769712] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2130.770526] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b10dfe22-7b49-4518-9a8e-60138af69298 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2130.792823] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Reconfiguring VM instance instance-00000048 to attach disk [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2130.793916] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-440d144c-65b4-439d-bac7-35daed609ea2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2130.817738] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2130.817738] env[62506]: value = "task-2190880" [ 2130.817738] env[62506]: _type = "Task" [ 2130.817738] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2130.827008] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190880, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2131.117691] env[62506]: DEBUG oslo_concurrency.lockutils [None req-1d8ed97d-bc25-4800-b6b9-adcc5c535b2d tempest-ServerGroupTestJSON-875031128 tempest-ServerGroupTestJSON-875031128-project-member] Lock "caa77bc8-f831-418e-bb46-bd7d5ed6ef2f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 86.307s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2131.131946] env[62506]: INFO nova.compute.manager [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] [instance: 6d89deb7-cc37-4feb-a998-fef1d8a92947] Took 1.02 seconds to deallocate network for instance. [ 2131.266889] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2131.331094] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190880, 'name': ReconfigVM_Task, 'duration_secs': 0.502899} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2131.331384] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Reconfigured VM instance instance-00000048 to attach disk [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2131.332027] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-654170e8-28e8-4af6-9f24-8376bc894562 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2131.345755] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2131.345755] env[62506]: value = "task-2190881" [ 2131.345755] env[62506]: _type = "Task" [ 2131.345755] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2131.353649] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190881, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2131.354595] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2131.412825] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-622bc1b6-36af-4ce9-9913-19391a013f23 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2131.420100] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69b5f22f-473e-4423-839d-70f2d9407809 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2131.451979] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5587d30f-1bc4-4859-adf8-89d061457952 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2131.459731] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a02059de-7138-4027-b4b4-d688f32b5753 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2131.473226] env[62506]: DEBUG nova.compute.provider_tree [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2131.620411] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2131.856113] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190881, 'name': Rename_Task, 'duration_secs': 0.128796} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2131.856350] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2131.856587] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-c45aeca9-5bd0-494b-812d-c3f882b6adae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2131.859653] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-91925f54-176b-4f34-8634-46bf7cb73bdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2131.859919] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2131.860222] env[62506]: DEBUG nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2131.860222] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2131.863997] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2131.863997] env[62506]: value = "task-2190882" [ 2131.863997] env[62506]: _type = "Task" [ 2131.863997] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2131.871800] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190882, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2131.879188] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2131.976654] env[62506]: DEBUG nova.scheduler.client.report [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2132.144866] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2132.165877] env[62506]: INFO nova.scheduler.client.report [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Deleted allocations for instance 6d89deb7-cc37-4feb-a998-fef1d8a92947 [ 2132.373794] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190882, 'name': PowerOnVM_Task} progress is 89%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2132.381382] env[62506]: DEBUG nova.network.neutron [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2132.482009] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.733s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2132.482710] env[62506]: ERROR nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Traceback (most recent call last): [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self.driver.spawn(context, instance, image_meta, [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] vm_ref = self.build_virtual_machine(instance, [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] vif_infos = vmwarevif.get_vif_info(self._session, [ 2132.482710] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] for vif in network_info: [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] return self._sync_wrapper(fn, *args, **kwargs) [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self.wait() [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self[:] = self._gt.wait() [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] return self._exit_event.wait() [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] current.throw(*self._exc) [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2132.483024] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] result = function(*args, **kwargs) [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] return func(*args, **kwargs) [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] raise e [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] nwinfo = self.network_api.allocate_for_instance( [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] created_port_ids = self._update_ports_for_instance( [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] with excutils.save_and_reraise_exception(): [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] self.force_reraise() [ 2132.483328] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] raise self.value [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] updated_port = self._update_port( [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] _ensure_no_port_binding_failure(port) [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] raise exception.PortBindingFailed(port_id=port['id']) [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] nova.exception.PortBindingFailed: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. [ 2132.483630] env[62506]: ERROR nova.compute.manager [instance: 80c76c85-2575-4733-a86a-111d138d69d4] [ 2132.483630] env[62506]: DEBUG nova.compute.utils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2132.485462] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Build of instance 80c76c85-2575-4733-a86a-111d138d69d4 was re-scheduled: Binding failed for port 38127312-6553-4ab8-8e8a-ebf3fe171bf7, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2132.485874] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2132.486127] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2132.486284] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquired lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2132.486442] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2132.487816] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.518s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2132.491080] env[62506]: INFO nova.compute.claims [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2132.673707] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e5988e1e-3598-404a-a2b8-47ff4edbf0a6 tempest-ImagesTestJSON-423920577 tempest-ImagesTestJSON-423920577-project-member] Lock "6d89deb7-cc37-4feb-a998-fef1d8a92947" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 84.216s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2132.876855] env[62506]: DEBUG oslo_vmware.api [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190882, 'name': PowerOnVM_Task, 'duration_secs': 0.561968} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2132.877031] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2132.877118] env[62506]: INFO nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Took 5.33 seconds to spawn the instance on the hypervisor. [ 2132.877276] env[62506]: DEBUG nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2132.879325] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f083de0-8fc4-4e50-a5bc-a53c28d34f55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2132.886288] env[62506]: INFO nova.compute.manager [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 91925f54-176b-4f34-8634-46bf7cb73bdd] Took 1.03 seconds to deallocate network for instance. [ 2133.007567] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2133.117882] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2133.176713] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2133.406146] env[62506]: INFO nova.compute.manager [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Took 26.29 seconds to build instance. [ 2133.620859] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Releasing lock "refresh_cache-80c76c85-2575-4733-a86a-111d138d69d4" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2133.621114] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2133.621307] env[62506]: DEBUG nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2133.621477] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2133.628228] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6dd11ce2-a2cd-4089-ab8c-4c1262bf2856 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2133.636195] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b803188-828d-4c7a-931e-399313d408c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2133.639636] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2133.668098] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6d2a257-2bc3-4731-b13c-188855ca5451 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2133.675864] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a25cdf9a-58d6-4a07-89a8-023f91e6eceb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2133.693401] env[62506]: DEBUG nova.compute.provider_tree [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2133.701865] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2133.912162] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51d6cba0-dc9a-4554-a60b-b2c77dafc419 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 48.691s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2133.938725] env[62506]: INFO nova.scheduler.client.report [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Deleted allocations for instance 91925f54-176b-4f34-8634-46bf7cb73bdd [ 2134.142493] env[62506]: DEBUG nova.network.neutron [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2134.196009] env[62506]: DEBUG nova.scheduler.client.report [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2134.448389] env[62506]: DEBUG oslo_concurrency.lockutils [None req-edda2c4e-33a8-4516-bf06-334dd8c2f890 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "91925f54-176b-4f34-8634-46bf7cb73bdd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 83.383s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2134.582420] env[62506]: INFO nova.compute.manager [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Rebuilding instance [ 2134.626343] env[62506]: DEBUG nova.compute.manager [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2134.627227] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5739b0fc-b234-4c76-99bf-0188b9dc28bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2134.647650] env[62506]: INFO nova.compute.manager [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: 80c76c85-2575-4733-a86a-111d138d69d4] Took 1.03 seconds to deallocate network for instance. [ 2134.702049] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.214s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2134.702583] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2134.705122] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.284s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2135.210340] env[62506]: DEBUG nova.compute.utils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2135.214820] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2135.214959] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2135.288500] env[62506]: DEBUG nova.policy [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4d5b44f1856745d6bdca5ed7829d1a44', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bd95448919b446be92ee08ed1e17a8a1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2135.352623] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9c93c39-8c5c-4845-a77c-7112698d2de5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2135.361643] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dc17061-bb78-4b27-bc05-6400d63e8538 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2135.393071] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7df743e-8f8c-45fa-80f1-f8077951d012 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2135.400749] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22327303-d6b4-41bc-b00f-7948caa47aa6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2135.414683] env[62506]: DEBUG nova.compute.provider_tree [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2135.645169] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Successfully created port: 4a72469c-bd76-46ca-b313-a0f83c357071 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2135.645169] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2135.645169] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-31144cfc-179e-403c-81e2-be7cb011ec22 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2135.655016] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2135.655016] env[62506]: value = "task-2190883" [ 2135.655016] env[62506]: _type = "Task" [ 2135.655016] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2135.665777] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190883, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2135.676269] env[62506]: INFO nova.scheduler.client.report [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Deleted allocations for instance 80c76c85-2575-4733-a86a-111d138d69d4 [ 2135.722127] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2135.917493] env[62506]: DEBUG nova.scheduler.client.report [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2135.978838] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Successfully created port: adb6781e-19cb-4d20-9984-1d5f482196a0 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2136.137050] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "7d013470-bf62-4db4-84f0-96179b417abf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2136.137050] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "7d013470-bf62-4db4-84f0-96179b417abf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2136.164770] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190883, 'name': PowerOffVM_Task, 'duration_secs': 0.143701} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2136.165360] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2136.165732] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2136.166617] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-546ecdba-f249-48ad-82f1-fa59d1c4ca93 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2136.175647] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2136.176634] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-959e48e1-30d1-4ce5-b4e2-0c94cee08c91 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2136.184810] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f039cada-0da5-4f7d-946a-ab962b32970e tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "80c76c85-2575-4733-a86a-111d138d69d4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 77.807s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2136.200941] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2136.201737] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2136.201737] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Deleting the datastore file [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2136.201737] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-234eb9da-106c-4ef2-9df4-4684f81597a2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2136.208703] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2136.208703] env[62506]: value = "task-2190885" [ 2136.208703] env[62506]: _type = "Task" [ 2136.208703] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2136.217638] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190885, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2136.290802] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Successfully created port: b465e67d-be27-4af4-9a3c-c6971af287e4 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2136.422200] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.717s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2136.422830] env[62506]: ERROR nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Traceback (most recent call last): [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self.driver.spawn(context, instance, image_meta, [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] vm_ref = self.build_virtual_machine(instance, [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] vif_infos = vmwarevif.get_vif_info(self._session, [ 2136.422830] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] for vif in network_info: [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] return self._sync_wrapper(fn, *args, **kwargs) [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self.wait() [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self[:] = self._gt.wait() [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] return self._exit_event.wait() [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] current.throw(*self._exc) [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2136.423179] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] result = function(*args, **kwargs) [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] return func(*args, **kwargs) [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] raise e [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] nwinfo = self.network_api.allocate_for_instance( [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] created_port_ids = self._update_ports_for_instance( [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] with excutils.save_and_reraise_exception(): [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] self.force_reraise() [ 2136.423536] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] raise self.value [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] updated_port = self._update_port( [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] _ensure_no_port_binding_failure(port) [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] raise exception.PortBindingFailed(port_id=port['id']) [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] nova.exception.PortBindingFailed: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. [ 2136.423931] env[62506]: ERROR nova.compute.manager [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] [ 2136.423931] env[62506]: DEBUG nova.compute.utils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2136.424827] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.868s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2136.431023] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Build of instance 1147abf9-da6d-4e51-878e-b8dc9a3f6141 was re-scheduled: Binding failed for port eb89e77d-ab13-4eb6-973b-a04d23eef0db, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2136.431023] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2136.431023] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2136.431023] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2136.431383] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2136.638941] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2136.720932] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190885, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.133011} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2136.721054] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2136.721187] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2136.721369] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2136.732408] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2136.771228] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2136.771474] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2136.771629] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2136.771810] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2136.772261] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2136.772261] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2136.772390] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2136.772555] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2136.772739] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2136.772920] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2136.773116] env[62506]: DEBUG nova.virt.hardware [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2136.774024] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cc659b6-b95d-4245-b9ec-26993aa909c5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2136.782540] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6465e418-1d2a-43bd-bce6-1cb1e6b92f21 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2136.982575] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2137.077848] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8658fce2-d38b-4078-9616-ad524b4f4bb1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.092172] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8117d41-adbe-4b5a-a269-df64a3367aec {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.129146] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4a0222e-2aff-40d3-bef0-b863d1a60882 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.138174] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8e0d89e-afc7-44a5-82ce-07f9c0b8115a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.157560] env[62506]: DEBUG nova.compute.provider_tree [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2137.164323] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2137.179406] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2137.286872] env[62506]: DEBUG nova.compute.manager [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Received event network-changed-4a72469c-bd76-46ca-b313-a0f83c357071 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2137.286872] env[62506]: DEBUG nova.compute.manager [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Refreshing instance network info cache due to event network-changed-4a72469c-bd76-46ca-b313-a0f83c357071. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2137.286872] env[62506]: DEBUG oslo_concurrency.lockutils [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] Acquiring lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2137.286872] env[62506]: DEBUG oslo_concurrency.lockutils [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] Acquired lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2137.286872] env[62506]: DEBUG nova.network.neutron [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Refreshing network info cache for port 4a72469c-bd76-46ca-b313-a0f83c357071 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2137.664512] env[62506]: DEBUG nova.scheduler.client.report [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2137.670168] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-1147abf9-da6d-4e51-878e-b8dc9a3f6141" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2137.670399] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2137.670581] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2137.670750] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2137.688334] env[62506]: ERROR nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2137.688334] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2137.688334] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2137.688334] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2137.688334] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2137.688334] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2137.688334] env[62506]: ERROR nova.compute.manager raise self.value [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2137.688334] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2137.688334] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2137.688334] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2137.688828] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2137.688828] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2137.688828] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2137.688828] env[62506]: ERROR nova.compute.manager [ 2137.688828] env[62506]: Traceback (most recent call last): [ 2137.688828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2137.688828] env[62506]: listener.cb(fileno) [ 2137.688828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2137.688828] env[62506]: result = function(*args, **kwargs) [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2137.688828] env[62506]: return func(*args, **kwargs) [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2137.688828] env[62506]: raise e [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2137.688828] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2137.688828] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2137.688828] env[62506]: with excutils.save_and_reraise_exception(): [ 2137.688828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2137.688828] env[62506]: self.force_reraise() [ 2137.688828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2137.688828] env[62506]: raise self.value [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2137.688828] env[62506]: updated_port = self._update_port( [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2137.688828] env[62506]: _ensure_no_port_binding_failure(port) [ 2137.688828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2137.688828] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2137.689683] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2137.689683] env[62506]: Removing descriptor: 20 [ 2137.689683] env[62506]: ERROR nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Traceback (most recent call last): [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] yield resources [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self.driver.spawn(context, instance, image_meta, [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2137.689683] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] vm_ref = self.build_virtual_machine(instance, [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] vif_infos = vmwarevif.get_vif_info(self._session, [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] for vif in network_info: [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return self._sync_wrapper(fn, *args, **kwargs) [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self.wait() [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self[:] = self._gt.wait() [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return self._exit_event.wait() [ 2137.690057] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] result = hub.switch() [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return self.greenlet.switch() [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] result = function(*args, **kwargs) [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return func(*args, **kwargs) [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] raise e [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] nwinfo = self.network_api.allocate_for_instance( [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2137.690435] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] created_port_ids = self._update_ports_for_instance( [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] with excutils.save_and_reraise_exception(): [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self.force_reraise() [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] raise self.value [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] updated_port = self._update_port( [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] _ensure_no_port_binding_failure(port) [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2137.690814] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] raise exception.PortBindingFailed(port_id=port['id']) [ 2137.691202] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2137.691202] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] [ 2137.691202] env[62506]: INFO nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Terminating instance [ 2137.700498] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2137.764063] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2137.764370] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2137.764533] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2137.764713] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2137.764860] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2137.765076] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2137.765276] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2137.765454] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2137.765627] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2137.765777] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2137.765944] env[62506]: DEBUG nova.virt.hardware [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2137.766826] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-281d9231-d78d-4db3-8c71-beb8acec9e10 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.775185] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-884dfbe8-004f-400a-8198-03f661e474c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.793882] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2137.799904] env[62506]: DEBUG oslo.service.loopingcall [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2137.799904] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2137.800917] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f1bb0e42-405f-4d80-9fbd-c593efa9e32f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.818495] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2137.818495] env[62506]: value = "task-2190886" [ 2137.818495] env[62506]: _type = "Task" [ 2137.818495] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2137.826783] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190886, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2137.863096] env[62506]: DEBUG nova.network.neutron [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2138.062615] env[62506]: DEBUG nova.network.neutron [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2138.175142] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.750s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2138.175797] env[62506]: ERROR nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Traceback (most recent call last): [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self.driver.spawn(context, instance, image_meta, [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] vm_ref = self.build_virtual_machine(instance, [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] vif_infos = vmwarevif.get_vif_info(self._session, [ 2138.175797] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] for vif in network_info: [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] return self._sync_wrapper(fn, *args, **kwargs) [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self.wait() [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self[:] = self._gt.wait() [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] return self._exit_event.wait() [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] current.throw(*self._exc) [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2138.176141] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] result = function(*args, **kwargs) [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] return func(*args, **kwargs) [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] raise e [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] nwinfo = self.network_api.allocate_for_instance( [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] created_port_ids = self._update_ports_for_instance( [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] with excutils.save_and_reraise_exception(): [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] self.force_reraise() [ 2138.176449] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] raise self.value [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] updated_port = self._update_port( [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] _ensure_no_port_binding_failure(port) [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] raise exception.PortBindingFailed(port_id=port['id']) [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] nova.exception.PortBindingFailed: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. [ 2138.176755] env[62506]: ERROR nova.compute.manager [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] [ 2138.176755] env[62506]: DEBUG nova.compute.utils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2138.180759] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Build of instance d4e49d27-403b-4f85-95b7-ae07e9bb07b8 was re-scheduled: Binding failed for port ec1d4c39-466a-49b1-a469-49fd06de3148, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2138.181209] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2138.181454] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquiring lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2138.181605] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Acquired lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2138.181764] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2138.183316] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.904s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2138.188014] env[62506]: INFO nova.compute.claims [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2138.193239] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2138.203477] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2138.333100] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190886, 'name': CreateVM_Task} progress is 25%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2138.565518] env[62506]: DEBUG oslo_concurrency.lockutils [req-2f016b9e-9290-4316-a12b-e8e94e2bf200 req-b81d08bd-208d-4299-a5ec-e91f65a01be1 service nova] Releasing lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2138.565944] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquired lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2138.566150] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2138.706027] env[62506]: INFO nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: 1147abf9-da6d-4e51-878e-b8dc9a3f6141] Took 1.03 seconds to deallocate network for instance. [ 2138.708591] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2138.783496] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2138.830540] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190886, 'name': CreateVM_Task} progress is 99%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2138.895932] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "a2735381-e738-419a-bf49-16459efbb439" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2138.896272] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "a2735381-e738-419a-bf49-16459efbb439" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2139.101631] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2139.206285] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "abcd5008-a127-4def-a29c-5bb8af85e196" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2139.206725] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "abcd5008-a127-4def-a29c-5bb8af85e196" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2139.233099] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2139.289330] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Releasing lock "refresh_cache-d4e49d27-403b-4f85-95b7-ae07e9bb07b8" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2139.289867] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2139.289867] env[62506]: DEBUG nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2139.289988] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2139.308443] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2139.315191] env[62506]: DEBUG nova.compute.manager [req-bb063273-69a7-47e0-9207-422cf5c2f41d req-bde559b3-9f57-4144-a972-a2d70580cd21 service nova] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Received event network-vif-deleted-4a72469c-bd76-46ca-b313-a0f83c357071 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2139.334745] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190886, 'name': CreateVM_Task, 'duration_secs': 1.070246} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2139.335035] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2139.335388] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2139.335551] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2139.335967] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2139.336244] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d490431c-9083-40bb-a8f8-24c459d3f786 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.345064] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2139.345064] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5207bef2-0e1f-2b51-b09e-c8974647a3cb" [ 2139.345064] env[62506]: _type = "Task" [ 2139.345064] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2139.361489] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5207bef2-0e1f-2b51-b09e-c8974647a3cb, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2139.387105] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1a30bce-ca09-4225-be10-eda7d26f80a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.394864] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-908bb7fd-b5a4-4d8d-a792-465616d30ba6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.398877] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2139.429744] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6964ca24-074a-4c70-ad24-834eb4614d9d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.437764] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcf60465-aa6a-4e42-a0b9-b6f1ad0ed4a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.452304] env[62506]: DEBUG nova.compute.provider_tree [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2139.709146] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2139.738507] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Releasing lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2139.739641] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2139.739641] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2139.739890] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-613d1035-dae6-43e6-980a-30b25ef0edce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.745256] env[62506]: INFO nova.scheduler.client.report [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Deleted allocations for instance 1147abf9-da6d-4e51-878e-b8dc9a3f6141 [ 2139.765442] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0289afb3-89ca-4318-8454-454866f53a3e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.792030] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ec283c13-1c5f-4dff-9254-3c278d87935a could not be found. [ 2139.794027] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2139.794027] env[62506]: INFO nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Took 0.05 seconds to destroy the instance on the hypervisor. [ 2139.794027] env[62506]: DEBUG oslo.service.loopingcall [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2139.794027] env[62506]: DEBUG nova.compute.manager [-] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2139.794027] env[62506]: DEBUG nova.network.neutron [-] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2139.811052] env[62506]: DEBUG nova.network.neutron [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2139.855946] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5207bef2-0e1f-2b51-b09e-c8974647a3cb, 'name': SearchDatastore_Task, 'duration_secs': 0.017278} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2139.856279] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2139.856518] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2139.856869] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2139.857061] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2139.857248] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2139.857511] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-03340b29-4eeb-4164-910d-a9fd24929c2d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.859874] env[62506]: DEBUG nova.network.neutron [-] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2139.868826] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2139.869027] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2139.869849] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a89b6f68-6457-4e58-b2e5-e865434fc720 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2139.875376] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2139.875376] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]520fddb9-ffd4-7e82-5f07-cae69fdfae75" [ 2139.875376] env[62506]: _type = "Task" [ 2139.875376] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2139.885467] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]520fddb9-ffd4-7e82-5f07-cae69fdfae75, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2139.921781] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2139.956111] env[62506]: DEBUG nova.scheduler.client.report [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2140.232191] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2140.257728] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "1147abf9-da6d-4e51-878e-b8dc9a3f6141" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 63.381s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2140.314395] env[62506]: INFO nova.compute.manager [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] [instance: d4e49d27-403b-4f85-95b7-ae07e9bb07b8] Took 1.02 seconds to deallocate network for instance. [ 2140.386663] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]520fddb9-ffd4-7e82-5f07-cae69fdfae75, 'name': SearchDatastore_Task, 'duration_secs': 0.008591} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2140.387481] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1b39ee07-ab32-4d54-8722-ba96581fea5e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2140.393108] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2140.393108] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5233cd45-9f6c-bdb9-e9ca-84c29662f434" [ 2140.393108] env[62506]: _type = "Task" [ 2140.393108] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2140.400894] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5233cd45-9f6c-bdb9-e9ca-84c29662f434, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2140.463278] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.278s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2140.463278] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2140.464630] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 13.146s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2140.747189] env[62506]: DEBUG nova.network.neutron [-] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2140.903618] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5233cd45-9f6c-bdb9-e9ca-84c29662f434, 'name': SearchDatastore_Task, 'duration_secs': 0.013511} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2140.903923] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2140.904208] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2140.904469] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a2184b90-8539-44c5-a718-ed1497890b96 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2140.910576] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2140.910576] env[62506]: value = "task-2190887" [ 2140.910576] env[62506]: _type = "Task" [ 2140.910576] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2140.918120] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190887, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2140.971817] env[62506]: DEBUG nova.compute.utils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2140.975305] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2140.975470] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2141.022251] env[62506]: DEBUG nova.policy [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ab67b57f49ad47ea9aab58b4f7bc563f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c97e56471ed471aa4a0812ec92e082c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2141.251103] env[62506]: INFO nova.compute.manager [-] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Took 1.46 seconds to deallocate network for instance. [ 2141.252178] env[62506]: DEBUG nova.compute.claims [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2141.252529] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2141.364105] env[62506]: INFO nova.scheduler.client.report [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Deleted allocations for instance d4e49d27-403b-4f85-95b7-ae07e9bb07b8 [ 2141.421400] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190887, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.483603} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2141.422302] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2141.422595] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2141.422907] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-7dc7b51c-b88c-4637-be6a-a2fc2d5f38b4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2141.430045] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2141.430045] env[62506]: value = "task-2190888" [ 2141.430045] env[62506]: _type = "Task" [ 2141.430045] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2141.438344] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190888, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2141.482282] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2141.506061] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2141.506061] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance ec283c13-1c5f-4dff-9254-3c278d87935a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2141.506061] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance ed482516-f639-4f5d-8e79-cdb72c85e304 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2141.560809] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Successfully created port: c6d12d9e-9e68-433e-a3e2-3e0560c3b30f {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2141.875011] env[62506]: DEBUG oslo_concurrency.lockutils [None req-eb3bc2d5-e4a9-41ca-9dba-bfd851fae960 tempest-MultipleCreateTestJSON-1420668454 tempest-MultipleCreateTestJSON-1420668454-project-member] Lock "d4e49d27-403b-4f85-95b7-ae07e9bb07b8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 64.970s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2141.942192] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190888, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.065833} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2141.942464] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2141.943263] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b70d4188-bc6b-443d-b844-64da86016fbd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2141.962428] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Reconfiguring VM instance instance-00000048 to attach disk [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2141.962699] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-22260a73-3e5a-47c1-a424-a2765afcbebb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2141.983333] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2141.983333] env[62506]: value = "task-2190889" [ 2141.983333] env[62506]: _type = "Task" [ 2141.983333] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2141.995926] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190889, 'name': ReconfigVM_Task} progress is 10%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2142.009110] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 220ccae9-6d65-4925-9fef-d555cb923fdd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2142.280216] env[62506]: DEBUG nova.compute.manager [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Received event network-changed-c6d12d9e-9e68-433e-a3e2-3e0560c3b30f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2142.280426] env[62506]: DEBUG nova.compute.manager [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Refreshing instance network info cache due to event network-changed-c6d12d9e-9e68-433e-a3e2-3e0560c3b30f. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2142.280642] env[62506]: DEBUG oslo_concurrency.lockutils [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] Acquiring lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2142.280834] env[62506]: DEBUG oslo_concurrency.lockutils [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] Acquired lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2142.280988] env[62506]: DEBUG nova.network.neutron [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Refreshing network info cache for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2142.439376] env[62506]: ERROR nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2142.439376] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2142.439376] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2142.439376] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2142.439376] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2142.439376] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2142.439376] env[62506]: ERROR nova.compute.manager raise self.value [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2142.439376] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2142.439376] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2142.439376] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2142.440089] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2142.440089] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2142.440089] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2142.440089] env[62506]: ERROR nova.compute.manager [ 2142.440089] env[62506]: Traceback (most recent call last): [ 2142.440089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2142.440089] env[62506]: listener.cb(fileno) [ 2142.440089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2142.440089] env[62506]: result = function(*args, **kwargs) [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2142.440089] env[62506]: return func(*args, **kwargs) [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2142.440089] env[62506]: raise e [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2142.440089] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2142.440089] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2142.440089] env[62506]: with excutils.save_and_reraise_exception(): [ 2142.440089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2142.440089] env[62506]: self.force_reraise() [ 2142.440089] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2142.440089] env[62506]: raise self.value [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2142.440089] env[62506]: updated_port = self._update_port( [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2142.440089] env[62506]: _ensure_no_port_binding_failure(port) [ 2142.440089] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2142.440089] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2142.440947] env[62506]: nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2142.440947] env[62506]: Removing descriptor: 19 [ 2142.494397] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2142.496404] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190889, 'name': ReconfigVM_Task, 'duration_secs': 0.290139} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2142.496836] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Reconfigured VM instance instance-00000048 to attach disk [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9/8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2142.497646] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-9b50a654-2091-4e5a-826b-353103d3d73b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.504662] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2142.504662] env[62506]: value = "task-2190890" [ 2142.504662] env[62506]: _type = "Task" [ 2142.504662] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2142.515135] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance ed4f2610-e5dd-4c56-bcc0-71ec198f4833 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2142.516351] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190890, 'name': Rename_Task} progress is 5%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2142.525332] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2142.525557] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2142.525786] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2142.525890] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2142.526092] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2142.526305] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2142.526580] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2142.526757] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2142.526957] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2142.527292] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2142.527489] env[62506]: DEBUG nova.virt.hardware [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2142.528418] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c57f1b5b-ad6f-4613-907e-7f525b4554f8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.536805] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3432656b-cca8-4bea-91cb-c680c0cec1ae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2142.551076] env[62506]: ERROR nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Traceback (most recent call last): [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] yield resources [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self.driver.spawn(context, instance, image_meta, [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] vm_ref = self.build_virtual_machine(instance, [ 2142.551076] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] vif_infos = vmwarevif.get_vif_info(self._session, [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] for vif in network_info: [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] return self._sync_wrapper(fn, *args, **kwargs) [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self.wait() [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self[:] = self._gt.wait() [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] return self._exit_event.wait() [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2142.551462] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] current.throw(*self._exc) [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] result = function(*args, **kwargs) [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] return func(*args, **kwargs) [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] raise e [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] nwinfo = self.network_api.allocate_for_instance( [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] created_port_ids = self._update_ports_for_instance( [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] with excutils.save_and_reraise_exception(): [ 2142.551783] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self.force_reraise() [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] raise self.value [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] updated_port = self._update_port( [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] _ensure_no_port_binding_failure(port) [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] raise exception.PortBindingFailed(port_id=port['id']) [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2142.552123] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] [ 2142.552123] env[62506]: INFO nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Terminating instance [ 2142.815716] env[62506]: DEBUG nova.network.neutron [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2142.959038] env[62506]: DEBUG nova.network.neutron [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2143.014641] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190890, 'name': Rename_Task, 'duration_secs': 0.172409} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2143.014867] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2143.015141] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-cfc4253d-93b4-42d1-bff7-81685e1550fc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2143.019494] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 99973068-d1a5-4809-b722-b6dd24ad3a12 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2143.021736] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2143.021736] env[62506]: value = "task-2190891" [ 2143.021736] env[62506]: _type = "Task" [ 2143.021736] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2143.030388] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190891, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2143.055237] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2143.464711] env[62506]: DEBUG oslo_concurrency.lockutils [req-5aae5d50-9040-420a-9134-0d4afb51fbc9 req-0745d3a4-147d-4514-8ca9-bb2df8e3465b service nova] Releasing lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2143.465282] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2143.466358] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2143.526410] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 7d013470-bf62-4db4-84f0-96179b417abf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2143.533244] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190891, 'name': PowerOnVM_Task} progress is 94%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2143.985665] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2144.033291] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance a2735381-e738-419a-bf49-16459efbb439 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2144.035199] env[62506]: DEBUG oslo_vmware.api [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190891, 'name': PowerOnVM_Task, 'duration_secs': 0.889364} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2144.035561] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2144.035784] env[62506]: DEBUG nova.compute.manager [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2144.036583] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e73fe6-37d9-48ca-ae02-315abd74ae88 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.107422] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2144.312306] env[62506]: DEBUG nova.compute.manager [req-547bb729-83b4-410d-a968-48b1cb95b349 req-91f401fa-8f50-4178-af40-1158a8b1ee2f service nova] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Received event network-vif-deleted-c6d12d9e-9e68-433e-a3e2-3e0560c3b30f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2144.537727] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance abcd5008-a127-4def-a29c-5bb8af85e196 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2144.537727] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 3 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2144.537727] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1088MB phys_disk=200GB used_disk=3GB total_vcpus=48 used_vcpus=3 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2144.552771] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2144.610299] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2144.610718] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2144.610912] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2144.611218] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c3885948-f197-4581-8552-69f93ba84af8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.620329] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ef51eec-2993-47a2-b10d-2975c9e2ebed {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.647843] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ed482516-f639-4f5d-8e79-cdb72c85e304 could not be found. [ 2144.648042] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2144.648240] env[62506]: INFO nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2144.648481] env[62506]: DEBUG oslo.service.loopingcall [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2144.648703] env[62506]: DEBUG nova.compute.manager [-] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2144.648799] env[62506]: DEBUG nova.network.neutron [-] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2144.666627] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c453c216-aa5d-4048-a5ac-965a1d77e182 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.670694] env[62506]: DEBUG nova.network.neutron [-] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2144.674470] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56f9a6a2-ab1b-403f-b512-147120f2b1e2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.703300] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88001c99-6cec-4c21-a5b4-ca23d7aaa613 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.710597] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e66f161-b7d4-4115-b5ea-8301c1a29287 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2144.724618] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2145.175824] env[62506]: DEBUG nova.network.neutron [-] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2145.228713] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2145.515444] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2145.515730] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2145.515951] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2145.516190] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2145.516367] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2145.518821] env[62506]: INFO nova.compute.manager [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Terminating instance [ 2145.676961] env[62506]: INFO nova.compute.manager [-] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Took 1.03 seconds to deallocate network for instance. [ 2145.679336] env[62506]: DEBUG nova.compute.claims [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2145.679581] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2145.733604] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2145.733922] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 5.269s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2145.734249] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.748s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2145.735848] env[62506]: INFO nova.compute.claims [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2146.024241] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "refresh_cache-8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2146.024426] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquired lock "refresh_cache-8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2146.024608] env[62506]: DEBUG nova.network.neutron [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2146.543366] env[62506]: DEBUG nova.network.neutron [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2146.603856] env[62506]: DEBUG nova.network.neutron [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2146.717612] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2146.717612] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2146.717612] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 2146.717983] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 2146.869052] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-679b3c1a-06f7-46dc-be92-f3054aba3d7f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2146.877121] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56fb3ece-d7b0-4efb-a21e-1fba0bc5ddcc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2146.908027] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82862075-7486-48a4-a07a-55adf802289c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2146.916385] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb46a7e4-3004-4094-bc0a-d17e415adc40 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2146.925090] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquiring lock "ba21b68c-eeae-4758-b65b-74b9780dbfcc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2146.925333] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Lock "ba21b68c-eeae-4758-b65b-74b9780dbfcc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2146.934558] env[62506]: DEBUG nova.compute.provider_tree [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2147.106530] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Releasing lock "refresh_cache-8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2147.106956] env[62506]: DEBUG nova.compute.manager [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2147.107569] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2147.108236] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98b1c67f-b7cb-4e82-93fd-7ded42b160ce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2147.116127] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2147.116358] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-801c008f-bffd-493e-9711-a8bc7ddd4310 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2147.122274] env[62506]: DEBUG oslo_vmware.api [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2147.122274] env[62506]: value = "task-2190892" [ 2147.122274] env[62506]: _type = "Task" [ 2147.122274] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2147.130122] env[62506]: DEBUG oslo_vmware.api [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190892, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2147.221646] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Skipping network cache update for instance because it is being deleted. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10462}} [ 2147.221833] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2147.221968] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2147.222108] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2147.222248] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 2147.222467] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2147.222629] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2147.222775] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2147.222963] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2147.223509] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 2147.437484] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2147.440650] env[62506]: DEBUG nova.scheduler.client.report [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2147.632573] env[62506]: DEBUG oslo_vmware.api [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190892, 'name': PowerOffVM_Task, 'duration_secs': 0.124961} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2147.632920] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2147.633128] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2147.633386] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-e552101f-432f-4dfe-adaa-3089be579463 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2147.657012] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2147.657257] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2147.657436] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Deleting the datastore file [datastore2] 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9 {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2147.657710] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ddb0aa81-231d-4907-bf45-d0a4f83f8ffd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2147.664132] env[62506]: DEBUG oslo_vmware.api [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for the task: (returnval){ [ 2147.664132] env[62506]: value = "task-2190894" [ 2147.664132] env[62506]: _type = "Task" [ 2147.664132] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2147.671797] env[62506]: DEBUG oslo_vmware.api [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190894, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2147.948149] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.214s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2147.948645] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2147.953126] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.809s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2147.954441] env[62506]: INFO nova.compute.claims [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2147.970665] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2148.174567] env[62506]: DEBUG oslo_vmware.api [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Task: {'id': task-2190894, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.108156} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2148.174567] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2148.174567] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2148.174567] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2148.174833] env[62506]: INFO nova.compute.manager [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Took 1.07 seconds to destroy the instance on the hypervisor. [ 2148.175145] env[62506]: DEBUG oslo.service.loopingcall [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2148.175370] env[62506]: DEBUG nova.compute.manager [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2148.175468] env[62506]: DEBUG nova.network.neutron [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2148.191387] env[62506]: DEBUG nova.network.neutron [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2148.459269] env[62506]: DEBUG nova.compute.utils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2148.462420] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2148.462590] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2148.524797] env[62506]: DEBUG nova.policy [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9389401046d64f4fa23f51a9fbefc899', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0fe07414118c4d82b70966bfb7b186ed', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2148.693680] env[62506]: DEBUG nova.network.neutron [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2148.833021] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Successfully created port: a3bed0c8-f014-4c57-a2ee-38cfed21f155 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2148.963537] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2149.112405] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ddc3f3a-6ba5-4072-8036-97026e25dc95 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2149.119807] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c558248-fd0d-4a3a-90f9-4b587e792170 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2149.149522] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40acd3f9-1f93-42bc-8a1a-8ff1f3f1d509 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2149.156988] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43504437-69c7-4a59-92d0-e976c662aa2c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2149.171170] env[62506]: DEBUG nova.compute.provider_tree [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2149.196239] env[62506]: INFO nova.compute.manager [-] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Took 1.02 seconds to deallocate network for instance. [ 2149.580870] env[62506]: DEBUG nova.compute.manager [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Received event network-changed-a3bed0c8-f014-4c57-a2ee-38cfed21f155 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2149.580959] env[62506]: DEBUG nova.compute.manager [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Refreshing instance network info cache due to event network-changed-a3bed0c8-f014-4c57-a2ee-38cfed21f155. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2149.581234] env[62506]: DEBUG oslo_concurrency.lockutils [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] Acquiring lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2149.581323] env[62506]: DEBUG oslo_concurrency.lockutils [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] Acquired lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2149.581612] env[62506]: DEBUG nova.network.neutron [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Refreshing network info cache for port a3bed0c8-f014-4c57-a2ee-38cfed21f155 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2149.677646] env[62506]: DEBUG nova.scheduler.client.report [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2149.701740] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2149.723704] env[62506]: ERROR nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2149.723704] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2149.723704] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2149.723704] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2149.723704] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2149.723704] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2149.723704] env[62506]: ERROR nova.compute.manager raise self.value [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2149.723704] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2149.723704] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2149.723704] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2149.724224] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2149.724224] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2149.724224] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2149.724224] env[62506]: ERROR nova.compute.manager [ 2149.724224] env[62506]: Traceback (most recent call last): [ 2149.724224] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2149.724224] env[62506]: listener.cb(fileno) [ 2149.724224] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2149.724224] env[62506]: result = function(*args, **kwargs) [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2149.724224] env[62506]: return func(*args, **kwargs) [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2149.724224] env[62506]: raise e [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2149.724224] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2149.724224] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2149.724224] env[62506]: with excutils.save_and_reraise_exception(): [ 2149.724224] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2149.724224] env[62506]: self.force_reraise() [ 2149.724224] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2149.724224] env[62506]: raise self.value [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2149.724224] env[62506]: updated_port = self._update_port( [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2149.724224] env[62506]: _ensure_no_port_binding_failure(port) [ 2149.724224] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2149.724224] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2149.725064] env[62506]: nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2149.725064] env[62506]: Removing descriptor: 19 [ 2149.979130] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2150.003319] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2150.003588] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2150.003762] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2150.004016] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2150.004181] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2150.004352] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2150.004628] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2150.004828] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2150.005042] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2150.005289] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2150.005518] env[62506]: DEBUG nova.virt.hardware [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2150.006443] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e2117d2-b1de-41f7-957b-45e910add21a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2150.014736] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34c0ccfd-ae39-46d4-b501-7fb51ed6e64b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2150.030638] env[62506]: ERROR nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Traceback (most recent call last): [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] yield resources [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self.driver.spawn(context, instance, image_meta, [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] vm_ref = self.build_virtual_machine(instance, [ 2150.030638] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] for vif in network_info: [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] return self._sync_wrapper(fn, *args, **kwargs) [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self.wait() [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self[:] = self._gt.wait() [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] return self._exit_event.wait() [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2150.031080] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] current.throw(*self._exc) [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] result = function(*args, **kwargs) [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] return func(*args, **kwargs) [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] raise e [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] nwinfo = self.network_api.allocate_for_instance( [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] created_port_ids = self._update_ports_for_instance( [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] with excutils.save_and_reraise_exception(): [ 2150.031437] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self.force_reraise() [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] raise self.value [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] updated_port = self._update_port( [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] _ensure_no_port_binding_failure(port) [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] raise exception.PortBindingFailed(port_id=port['id']) [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2150.031805] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] [ 2150.031805] env[62506]: INFO nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Terminating instance [ 2150.098326] env[62506]: DEBUG nova.network.neutron [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2150.182516] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.229s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2150.183072] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2150.185664] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.484s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2150.187130] env[62506]: INFO nova.compute.claims [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2150.229848] env[62506]: DEBUG nova.network.neutron [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2150.535177] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2150.693676] env[62506]: DEBUG nova.compute.utils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2150.696154] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2150.696350] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2150.731821] env[62506]: DEBUG oslo_concurrency.lockutils [req-d91fde9f-53af-4e85-a009-5201d1cbfe9e req-7c3d5ddd-f8f3-4a03-b743-c6f237627e47 service nova] Releasing lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2150.732865] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquired lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2150.732865] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2150.734495] env[62506]: DEBUG nova.policy [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3c9c372663b34fcaa0dac3e1c657e044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fc10dc6a19fb41cdbf046f6da6dc69fe', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2151.035831] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Successfully created port: f08de356-f2ca-43cc-99c0-160d2cea8f2d {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2151.197336] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2151.257481] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2151.331911] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2151.335047] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-703a861f-58c3-4d9e-be20-ad957382882a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2151.343131] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a90a45b9-c21a-4e43-9d1c-bae0bd53fe44 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2151.374164] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6002c468-0540-431e-aa7c-91ae514d3f89 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2151.381581] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1078fdfc-98ca-4c57-a938-742077f3fd48 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2151.394736] env[62506]: DEBUG nova.compute.provider_tree [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2151.605126] env[62506]: DEBUG nova.compute.manager [req-49652d96-2f5c-4e18-93c6-6af69a0b95a1 req-7f336d67-2584-4520-b02b-ce20444fa2ee service nova] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Received event network-vif-deleted-a3bed0c8-f014-4c57-a2ee-38cfed21f155 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2151.838470] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Releasing lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2151.838902] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2151.839113] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2151.839775] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5989be40-221c-49de-ae4b-7f643105891c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2151.849633] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92e79ca1-6464-4fff-882b-56f26c8aaba6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2151.873121] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 220ccae9-6d65-4925-9fef-d555cb923fdd could not be found. [ 2151.873361] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2151.873536] env[62506]: INFO nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2151.873771] env[62506]: DEBUG oslo.service.loopingcall [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2151.873990] env[62506]: DEBUG nova.compute.manager [-] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2151.874115] env[62506]: DEBUG nova.network.neutron [-] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2151.889122] env[62506]: DEBUG nova.network.neutron [-] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2151.897805] env[62506]: DEBUG nova.scheduler.client.report [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2151.907419] env[62506]: ERROR nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2151.907419] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2151.907419] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2151.907419] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2151.907419] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2151.907419] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2151.907419] env[62506]: ERROR nova.compute.manager raise self.value [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2151.907419] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2151.907419] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2151.907419] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2151.908269] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2151.908269] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2151.908269] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2151.908269] env[62506]: ERROR nova.compute.manager [ 2151.908269] env[62506]: Traceback (most recent call last): [ 2151.908269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2151.908269] env[62506]: listener.cb(fileno) [ 2151.908269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2151.908269] env[62506]: result = function(*args, **kwargs) [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2151.908269] env[62506]: return func(*args, **kwargs) [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2151.908269] env[62506]: raise e [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2151.908269] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2151.908269] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2151.908269] env[62506]: with excutils.save_and_reraise_exception(): [ 2151.908269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2151.908269] env[62506]: self.force_reraise() [ 2151.908269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2151.908269] env[62506]: raise self.value [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2151.908269] env[62506]: updated_port = self._update_port( [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2151.908269] env[62506]: _ensure_no_port_binding_failure(port) [ 2151.908269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2151.908269] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2151.909912] env[62506]: nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2151.909912] env[62506]: Removing descriptor: 19 [ 2152.209503] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2152.234544] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2152.234803] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2152.234959] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2152.235160] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2152.235340] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2152.235493] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2152.235698] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2152.235855] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2152.236028] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2152.236193] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2152.236365] env[62506]: DEBUG nova.virt.hardware [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2152.237213] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f5bb956-2f6d-47d3-8b0d-d43de6de6e5a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2152.244920] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f901578-0efc-4df0-8398-19bda42e06c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2152.258245] env[62506]: ERROR nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Traceback (most recent call last): [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] yield resources [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self.driver.spawn(context, instance, image_meta, [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] vm_ref = self.build_virtual_machine(instance, [ 2152.258245] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] vif_infos = vmwarevif.get_vif_info(self._session, [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] for vif in network_info: [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] return self._sync_wrapper(fn, *args, **kwargs) [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self.wait() [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self[:] = self._gt.wait() [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] return self._exit_event.wait() [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2152.258632] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] current.throw(*self._exc) [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] result = function(*args, **kwargs) [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] return func(*args, **kwargs) [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] raise e [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] nwinfo = self.network_api.allocate_for_instance( [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] created_port_ids = self._update_ports_for_instance( [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] with excutils.save_and_reraise_exception(): [ 2152.259031] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self.force_reraise() [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] raise self.value [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] updated_port = self._update_port( [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] _ensure_no_port_binding_failure(port) [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] raise exception.PortBindingFailed(port_id=port['id']) [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2152.259424] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] [ 2152.259424] env[62506]: INFO nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Terminating instance [ 2152.391223] env[62506]: DEBUG nova.network.neutron [-] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2152.401824] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.216s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2152.402226] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2152.405044] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.227s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2152.406516] env[62506]: INFO nova.compute.claims [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2152.762248] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2152.762511] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2152.762702] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2152.893494] env[62506]: INFO nova.compute.manager [-] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Took 1.02 seconds to deallocate network for instance. [ 2152.895724] env[62506]: DEBUG nova.compute.claims [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2152.895898] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2152.910601] env[62506]: DEBUG nova.compute.utils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2152.914134] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2152.914134] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2152.966184] env[62506]: DEBUG nova.policy [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a885dcd7c52e4089966a915df9bd2252', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ce556c00954e4e9caefee6b252f971c6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2153.272591] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Successfully created port: f903a622-c2b3-4b0e-8f69-fb0c339ec506 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2153.279740] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2153.351570] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2153.415184] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2153.560828] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d980bd70-8937-42c3-ba9d-b426d36b64bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.568846] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-277647f4-b09e-4c84-898e-df228e225bd0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.600248] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a2347de-11d4-4b5d-b6c7-1c3d55609414 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.608311] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a09e62d9-febb-45e1-b1b4-0c949cceb700 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.622753] env[62506]: DEBUG nova.compute.provider_tree [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2153.630381] env[62506]: DEBUG nova.compute.manager [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Received event network-changed-f08de356-f2ca-43cc-99c0-160d2cea8f2d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2153.630533] env[62506]: DEBUG nova.compute.manager [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Refreshing instance network info cache due to event network-changed-f08de356-f2ca-43cc-99c0-160d2cea8f2d. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2153.630734] env[62506]: DEBUG oslo_concurrency.lockutils [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] Acquiring lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2153.854866] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2153.855288] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2153.855485] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2153.855802] env[62506]: DEBUG oslo_concurrency.lockutils [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] Acquired lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2153.855983] env[62506]: DEBUG nova.network.neutron [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Refreshing network info cache for port f08de356-f2ca-43cc-99c0-160d2cea8f2d {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2153.857233] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9ac61707-8ee5-4088-8f09-87ca6ba439c7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.867250] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8cdc42a-2741-4d52-9ee4-d851b29bb465 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.894058] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ed4f2610-e5dd-4c56-bcc0-71ec198f4833 could not be found. [ 2153.894285] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2153.894463] env[62506]: INFO nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2153.894699] env[62506]: DEBUG oslo.service.loopingcall [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2153.894920] env[62506]: DEBUG nova.compute.manager [-] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2153.895030] env[62506]: DEBUG nova.network.neutron [-] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2153.910051] env[62506]: DEBUG nova.network.neutron [-] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2153.922249] env[62506]: INFO nova.virt.block_device [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Booting with volume 13a51316-89a9-4289-884a-b15db4757de3 at /dev/sda [ 2153.958882] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3baa6c22-6f35-4ac6-9499-25b3f792ec4e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.967838] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea271a32-d4ef-4e02-b5bf-23b437f00625 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.993595] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-38956e38-fdb3-475e-90da-0fb20ce4eda1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.005095] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7a40bb4-c116-4ab0-90f6-725942d3664a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.029284] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91d7b0f2-4f1a-4c2e-9a23-16aac52bdcf6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.035976] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35174bdb-5d95-4d59-888d-1e37e44ae6d9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.051225] env[62506]: DEBUG nova.virt.block_device [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Updating existing volume attachment record: 1a6f832a-4105-4e60-889f-4b669c0da582 {{(pid=62506) _volume_attach /opt/stack/nova/nova/virt/block_device.py:666}} [ 2154.125927] env[62506]: DEBUG nova.scheduler.client.report [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2154.207131] env[62506]: ERROR nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2154.207131] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2154.207131] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2154.207131] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2154.207131] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2154.207131] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2154.207131] env[62506]: ERROR nova.compute.manager raise self.value [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2154.207131] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2154.207131] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2154.207131] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2154.207622] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2154.207622] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2154.207622] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2154.207622] env[62506]: ERROR nova.compute.manager [ 2154.207622] env[62506]: Traceback (most recent call last): [ 2154.207622] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2154.207622] env[62506]: listener.cb(fileno) [ 2154.207622] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2154.207622] env[62506]: result = function(*args, **kwargs) [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2154.207622] env[62506]: return func(*args, **kwargs) [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2154.207622] env[62506]: raise e [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2154.207622] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2154.207622] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2154.207622] env[62506]: with excutils.save_and_reraise_exception(): [ 2154.207622] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2154.207622] env[62506]: self.force_reraise() [ 2154.207622] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2154.207622] env[62506]: raise self.value [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2154.207622] env[62506]: updated_port = self._update_port( [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2154.207622] env[62506]: _ensure_no_port_binding_failure(port) [ 2154.207622] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2154.207622] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2154.208466] env[62506]: nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2154.208466] env[62506]: Removing descriptor: 19 [ 2154.377265] env[62506]: DEBUG nova.network.neutron [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2154.413229] env[62506]: DEBUG nova.network.neutron [-] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2154.448871] env[62506]: DEBUG nova.network.neutron [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2154.630706] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.226s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2154.631267] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2154.633841] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.712s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2154.635666] env[62506]: INFO nova.compute.claims [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2154.918259] env[62506]: INFO nova.compute.manager [-] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Took 1.02 seconds to deallocate network for instance. [ 2154.920651] env[62506]: DEBUG nova.compute.claims [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2154.920839] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2154.951932] env[62506]: DEBUG oslo_concurrency.lockutils [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] Releasing lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2154.952197] env[62506]: DEBUG nova.compute.manager [req-ea447ca1-a1b0-410c-b4b5-9995b0758643 req-4f7fbd5f-517e-4666-a0e3-7448007deaf9 service nova] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Received event network-vif-deleted-f08de356-f2ca-43cc-99c0-160d2cea8f2d {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2155.140558] env[62506]: DEBUG nova.compute.utils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2155.143453] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2155.143620] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2155.190927] env[62506]: DEBUG nova.policy [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e508d2263c0c47d5bccfb1e2f1409316', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7d771ac74ac54fb38a84337eed72348a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2155.492710] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Successfully created port: ed27e006-061c-4877-aef2-24bf727d77c4 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2155.644084] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2155.679765] env[62506]: DEBUG nova.compute.manager [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Received event network-changed-f903a622-c2b3-4b0e-8f69-fb0c339ec506 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2155.679765] env[62506]: DEBUG nova.compute.manager [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Refreshing instance network info cache due to event network-changed-f903a622-c2b3-4b0e-8f69-fb0c339ec506. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2155.679765] env[62506]: DEBUG oslo_concurrency.lockutils [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] Acquiring lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2155.679765] env[62506]: DEBUG oslo_concurrency.lockutils [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] Acquired lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2155.679765] env[62506]: DEBUG nova.network.neutron [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Refreshing network info cache for port f903a622-c2b3-4b0e-8f69-fb0c339ec506 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2155.788810] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36d9ae70-6b52-4824-8017-40d761a35dfa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2155.798213] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b887564-7997-4488-9a79-92818aa70a33 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2155.830013] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58ad0484-eec2-4dd4-bf99-c9e7a0f278eb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2155.837560] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd9a4f15-7bcc-46e0-81c5-ff05c77973d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2155.850754] env[62506]: DEBUG nova.compute.provider_tree [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2156.140936] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2156.141557] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2156.141773] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2156.141929] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2156.142132] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2156.142295] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2156.142419] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2156.142624] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2156.142783] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2156.142980] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2156.143192] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2156.143383] env[62506]: DEBUG nova.virt.hardware [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2156.144950] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ab59f30-db72-425d-bcc2-e248dbef331f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2156.156283] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c227912-2429-4826-be1b-e131c1873785 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2156.170565] env[62506]: ERROR nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Traceback (most recent call last): [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] yield resources [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self.driver.spawn(context, instance, image_meta, [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] vm_ref = self.build_virtual_machine(instance, [ 2156.170565] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] vif_infos = vmwarevif.get_vif_info(self._session, [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] for vif in network_info: [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] return self._sync_wrapper(fn, *args, **kwargs) [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self.wait() [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self[:] = self._gt.wait() [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] return self._exit_event.wait() [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2156.171043] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] current.throw(*self._exc) [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] result = function(*args, **kwargs) [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] return func(*args, **kwargs) [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] raise e [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] nwinfo = self.network_api.allocate_for_instance( [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] created_port_ids = self._update_ports_for_instance( [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] with excutils.save_and_reraise_exception(): [ 2156.171414] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self.force_reraise() [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] raise self.value [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] updated_port = self._update_port( [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] _ensure_no_port_binding_failure(port) [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] raise exception.PortBindingFailed(port_id=port['id']) [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2156.171753] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] [ 2156.171753] env[62506]: INFO nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Terminating instance [ 2156.200310] env[62506]: DEBUG nova.network.neutron [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2156.278187] env[62506]: DEBUG nova.network.neutron [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2156.356029] env[62506]: DEBUG nova.scheduler.client.report [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2156.560277] env[62506]: ERROR nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2156.560277] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2156.560277] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2156.560277] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2156.560277] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2156.560277] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2156.560277] env[62506]: ERROR nova.compute.manager raise self.value [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2156.560277] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2156.560277] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2156.560277] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2156.560990] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2156.560990] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2156.560990] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2156.560990] env[62506]: ERROR nova.compute.manager [ 2156.560990] env[62506]: Traceback (most recent call last): [ 2156.560990] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2156.560990] env[62506]: listener.cb(fileno) [ 2156.560990] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2156.560990] env[62506]: result = function(*args, **kwargs) [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2156.560990] env[62506]: return func(*args, **kwargs) [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2156.560990] env[62506]: raise e [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2156.560990] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2156.560990] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2156.560990] env[62506]: with excutils.save_and_reraise_exception(): [ 2156.560990] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2156.560990] env[62506]: self.force_reraise() [ 2156.560990] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2156.560990] env[62506]: raise self.value [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2156.560990] env[62506]: updated_port = self._update_port( [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2156.560990] env[62506]: _ensure_no_port_binding_failure(port) [ 2156.560990] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2156.560990] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2156.561880] env[62506]: nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2156.561880] env[62506]: Removing descriptor: 19 [ 2156.662595] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2156.675526] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquiring lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2156.688167] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2156.688439] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2156.688621] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2156.688811] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2156.688965] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2156.689153] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2156.689362] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2156.689520] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2156.689682] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2156.690382] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2156.690382] env[62506]: DEBUG nova.virt.hardware [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2156.690888] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a49bf9bf-47ef-4025-885f-a6196e1004c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2156.699397] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59b6029d-220b-487a-92a2-b6a91f5bdb01 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2156.713676] env[62506]: ERROR nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Traceback (most recent call last): [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] yield resources [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self.driver.spawn(context, instance, image_meta, [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] vm_ref = self.build_virtual_machine(instance, [ 2156.713676] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] vif_infos = vmwarevif.get_vif_info(self._session, [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] for vif in network_info: [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] return self._sync_wrapper(fn, *args, **kwargs) [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self.wait() [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self[:] = self._gt.wait() [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] return self._exit_event.wait() [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2156.714099] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] current.throw(*self._exc) [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] result = function(*args, **kwargs) [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] return func(*args, **kwargs) [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] raise e [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] nwinfo = self.network_api.allocate_for_instance( [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] created_port_ids = self._update_ports_for_instance( [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] with excutils.save_and_reraise_exception(): [ 2156.714512] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self.force_reraise() [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] raise self.value [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] updated_port = self._update_port( [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] _ensure_no_port_binding_failure(port) [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] raise exception.PortBindingFailed(port_id=port['id']) [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2156.714991] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] [ 2156.714991] env[62506]: INFO nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Terminating instance [ 2156.781420] env[62506]: DEBUG oslo_concurrency.lockutils [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] Releasing lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2156.781797] env[62506]: DEBUG nova.compute.manager [req-1989bdc3-7a28-407f-bca1-b8a276761b70 req-b315fb90-7dd2-475f-bb86-f045b3882d51 service nova] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Received event network-vif-deleted-f903a622-c2b3-4b0e-8f69-fb0c339ec506 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2156.782227] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquired lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2156.782477] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2156.862566] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.229s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2156.863053] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2156.865558] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.635s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2156.866919] env[62506]: INFO nova.compute.claims [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2157.218808] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2157.218808] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2157.218808] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2157.300789] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2157.371560] env[62506]: DEBUG nova.compute.utils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2157.374637] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2157.374735] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2157.390066] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2157.416346] env[62506]: DEBUG nova.policy [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '59f4559064a14dea9d74a4fef4eff6d7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '612309ae575f46bdb31861d769d09fd8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2157.699071] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Successfully created port: 53318629-e4cd-4143-abb7-2a7d0c776bf6 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2157.703615] env[62506]: DEBUG nova.compute.manager [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Received event network-changed-ed27e006-061c-4877-aef2-24bf727d77c4 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2157.703802] env[62506]: DEBUG nova.compute.manager [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Refreshing instance network info cache due to event network-changed-ed27e006-061c-4877-aef2-24bf727d77c4. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2157.703999] env[62506]: DEBUG oslo_concurrency.lockutils [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] Acquiring lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2157.738511] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2157.871890] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2157.879606] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2157.892564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Releasing lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2157.893289] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2157.893743] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a2523c68-c146-4fac-bf83-ce116effdefc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2157.904613] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfd78d05-da67-4eb4-9d01-59c06c6a2ce7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2157.930791] env[62506]: WARNING nova.virt.vmwareapi.driver [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 99973068-d1a5-4809-b722-b6dd24ad3a12 could not be found. [ 2157.931066] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2157.933552] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cea4ddf1-d8c9-4031-ac21-34aa14f6550e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2157.942201] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dc9d2fc-da92-4af4-ac29-701d09faa641 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2157.968292] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 99973068-d1a5-4809-b722-b6dd24ad3a12 could not be found. [ 2157.968534] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2157.968718] env[62506]: INFO nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Took 0.08 seconds to destroy the instance on the hypervisor. [ 2157.968995] env[62506]: DEBUG oslo.service.loopingcall [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2157.971748] env[62506]: DEBUG nova.compute.manager [-] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2157.971748] env[62506]: DEBUG nova.network.neutron [-] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2157.994126] env[62506]: DEBUG nova.network.neutron [-] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2158.059142] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df5a9497-2c44-4a0e-8204-aabd2c458e10 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.066704] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94db8e18-4faa-4210-838d-f74909a823ff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.096215] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de6e6449-2213-4adb-9acd-6a260ad61330 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.103670] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffccad63-96a0-42a7-8310-8052e93e56dd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.116447] env[62506]: DEBUG nova.compute.provider_tree [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2158.340806] env[62506]: DEBUG nova.compute.manager [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] [instance: a2735381-e738-419a-bf49-16459efbb439] Received event network-changed-53318629-e4cd-4143-abb7-2a7d0c776bf6 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2158.341011] env[62506]: DEBUG nova.compute.manager [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] [instance: a2735381-e738-419a-bf49-16459efbb439] Refreshing instance network info cache due to event network-changed-53318629-e4cd-4143-abb7-2a7d0c776bf6. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2158.341239] env[62506]: DEBUG oslo_concurrency.lockutils [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] Acquiring lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2158.341380] env[62506]: DEBUG oslo_concurrency.lockutils [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] Acquired lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2158.341533] env[62506]: DEBUG nova.network.neutron [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] [instance: a2735381-e738-419a-bf49-16459efbb439] Refreshing network info cache for port 53318629-e4cd-4143-abb7-2a7d0c776bf6 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2158.375034] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2158.375455] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2158.375645] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2158.376480] env[62506]: DEBUG oslo_concurrency.lockutils [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] Acquired lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2158.376663] env[62506]: DEBUG nova.network.neutron [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Refreshing network info cache for port ed27e006-061c-4877-aef2-24bf727d77c4 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2158.377606] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-aca260a5-247f-4960-86e6-4ab69b3b0d6d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.386801] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-462e7564-cf89-4a26-ad37-4a4f4fd44e78 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.414850] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7d013470-bf62-4db4-84f0-96179b417abf could not be found. [ 2158.415090] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2158.415291] env[62506]: INFO nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2158.415534] env[62506]: DEBUG oslo.service.loopingcall [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2158.415755] env[62506]: DEBUG nova.compute.manager [-] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2158.415852] env[62506]: DEBUG nova.network.neutron [-] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2158.431934] env[62506]: DEBUG nova.network.neutron [-] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2158.496949] env[62506]: DEBUG nova.network.neutron [-] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2158.501726] env[62506]: ERROR nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2158.501726] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2158.501726] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2158.501726] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2158.501726] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2158.501726] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2158.501726] env[62506]: ERROR nova.compute.manager raise self.value [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2158.501726] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2158.501726] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2158.501726] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2158.502423] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2158.502423] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2158.502423] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2158.502423] env[62506]: ERROR nova.compute.manager [ 2158.502423] env[62506]: Traceback (most recent call last): [ 2158.502423] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2158.502423] env[62506]: listener.cb(fileno) [ 2158.502423] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2158.502423] env[62506]: result = function(*args, **kwargs) [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2158.502423] env[62506]: return func(*args, **kwargs) [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2158.502423] env[62506]: raise e [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2158.502423] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2158.502423] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2158.502423] env[62506]: with excutils.save_and_reraise_exception(): [ 2158.502423] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2158.502423] env[62506]: self.force_reraise() [ 2158.502423] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2158.502423] env[62506]: raise self.value [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2158.502423] env[62506]: updated_port = self._update_port( [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2158.502423] env[62506]: _ensure_no_port_binding_failure(port) [ 2158.502423] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2158.502423] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2158.503665] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2158.503665] env[62506]: Removing descriptor: 20 [ 2158.619843] env[62506]: DEBUG nova.scheduler.client.report [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2158.859274] env[62506]: DEBUG nova.network.neutron [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2158.896349] env[62506]: DEBUG nova.network.neutron [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2158.903914] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2158.931144] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2158.931398] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2158.931557] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2158.931817] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2158.932156] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2158.932229] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2158.932406] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2158.932564] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2158.932727] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2158.932881] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2158.933239] env[62506]: DEBUG nova.virt.hardware [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2158.934146] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe9b8949-6309-4d3c-a02e-c4b858f2bf67 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.936860] env[62506]: DEBUG nova.network.neutron [-] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2158.943981] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d378f8b-6082-476f-af89-49d12be32b29 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2158.951539] env[62506]: DEBUG nova.network.neutron [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] [instance: a2735381-e738-419a-bf49-16459efbb439] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2158.964841] env[62506]: ERROR nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] Traceback (most recent call last): [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] yield resources [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self.driver.spawn(context, instance, image_meta, [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] vm_ref = self.build_virtual_machine(instance, [ 2158.964841] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] vif_infos = vmwarevif.get_vif_info(self._session, [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] for vif in network_info: [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] return self._sync_wrapper(fn, *args, **kwargs) [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self.wait() [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self[:] = self._gt.wait() [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] return self._exit_event.wait() [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2158.965270] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] current.throw(*self._exc) [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] result = function(*args, **kwargs) [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] return func(*args, **kwargs) [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] raise e [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] nwinfo = self.network_api.allocate_for_instance( [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] created_port_ids = self._update_ports_for_instance( [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] with excutils.save_and_reraise_exception(): [ 2158.965685] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self.force_reraise() [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] raise self.value [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] updated_port = self._update_port( [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] _ensure_no_port_binding_failure(port) [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] raise exception.PortBindingFailed(port_id=port['id']) [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2158.966254] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] [ 2158.966254] env[62506]: INFO nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Terminating instance [ 2158.993338] env[62506]: DEBUG nova.network.neutron [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2158.999083] env[62506]: INFO nova.compute.manager [-] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Took 1.03 seconds to deallocate network for instance. [ 2159.124564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.259s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2159.125129] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2159.127675] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.875s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2159.440113] env[62506]: INFO nova.compute.manager [-] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Took 1.02 seconds to deallocate network for instance. [ 2159.442994] env[62506]: DEBUG nova.compute.claims [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2159.443224] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2159.453850] env[62506]: DEBUG oslo_concurrency.lockutils [req-95fbbcea-0637-4869-8f16-50a02fbf77bb req-9e05d335-b79a-4fc8-a7fa-171a4aa8b573 service nova] Releasing lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2159.469694] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2159.469694] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquired lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2159.469694] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2159.495339] env[62506]: DEBUG oslo_concurrency.lockutils [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] Releasing lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2159.495596] env[62506]: DEBUG nova.compute.manager [req-6d3f0747-15f9-4df8-8074-21d261c82243 req-a4edbf23-a10b-4b17-bfb8-373ed5c1a7c6 service nova] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Received event network-vif-deleted-ed27e006-061c-4877-aef2-24bf727d77c4 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2159.541867] env[62506]: INFO nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Took 0.54 seconds to detach 1 volumes for instance. [ 2159.544172] env[62506]: DEBUG nova.compute.claims [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2159.544369] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2159.636593] env[62506]: DEBUG nova.compute.utils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2159.638051] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2159.638257] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2159.677924] env[62506]: DEBUG nova.policy [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '083d0aee5b9343b2992cb2020e3acef1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '92286cfe20c34cb1b8f6bf106098d032', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2159.754277] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2daf8c9b-6e1d-4644-8954-b57552158133 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2159.761872] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbbb12f7-7d8d-4954-b268-21cb8f788c6d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2159.791842] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25f5feba-fbdf-4be5-aa18-e4c902119c55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2159.799141] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-793428df-0d4a-4d66-95b5-97113f2d676d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2159.812352] env[62506]: DEBUG nova.compute.provider_tree [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2159.971718] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Successfully created port: 89e25f07-3613-4320-b89e-a52b896c4a8f {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2159.996276] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2160.068158] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2160.142725] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2160.315969] env[62506]: DEBUG nova.scheduler.client.report [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2160.369989] env[62506]: DEBUG nova.compute.manager [req-0171b74f-ceff-4a05-bab8-57032b2abf55 req-600512a8-3d95-46af-afe4-4467c40919e0 service nova] [instance: a2735381-e738-419a-bf49-16459efbb439] Received event network-vif-deleted-53318629-e4cd-4143-abb7-2a7d0c776bf6 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2160.571017] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Releasing lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2160.571017] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2160.571017] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2160.571017] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c1738d85-dcb2-48c4-af17-8f96f8c950bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2160.585096] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-651208b5-27e5-41fa-8279-8f447c475e4c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2160.608899] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a2735381-e738-419a-bf49-16459efbb439 could not be found. [ 2160.609146] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2160.609416] env[62506]: INFO nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2160.609597] env[62506]: DEBUG oslo.service.loopingcall [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2160.609832] env[62506]: DEBUG nova.compute.manager [-] [instance: a2735381-e738-419a-bf49-16459efbb439] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2160.609933] env[62506]: DEBUG nova.network.neutron [-] [instance: a2735381-e738-419a-bf49-16459efbb439] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2160.627765] env[62506]: DEBUG nova.network.neutron [-] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2160.712269] env[62506]: DEBUG nova.compute.manager [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Received event network-changed-89e25f07-3613-4320-b89e-a52b896c4a8f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2160.712549] env[62506]: DEBUG nova.compute.manager [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Refreshing instance network info cache due to event network-changed-89e25f07-3613-4320-b89e-a52b896c4a8f. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2160.712860] env[62506]: DEBUG oslo_concurrency.lockutils [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] Acquiring lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2160.713132] env[62506]: DEBUG oslo_concurrency.lockutils [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] Acquired lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2160.713309] env[62506]: DEBUG nova.network.neutron [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Refreshing network info cache for port 89e25f07-3613-4320-b89e-a52b896c4a8f {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2160.821199] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.693s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2160.821837] env[62506]: ERROR nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Traceback (most recent call last): [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self.driver.spawn(context, instance, image_meta, [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] vm_ref = self.build_virtual_machine(instance, [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] vif_infos = vmwarevif.get_vif_info(self._session, [ 2160.821837] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] for vif in network_info: [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return self._sync_wrapper(fn, *args, **kwargs) [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self.wait() [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self[:] = self._gt.wait() [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return self._exit_event.wait() [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] result = hub.switch() [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2160.822261] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return self.greenlet.switch() [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] result = function(*args, **kwargs) [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] return func(*args, **kwargs) [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] raise e [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] nwinfo = self.network_api.allocate_for_instance( [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] created_port_ids = self._update_ports_for_instance( [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] with excutils.save_and_reraise_exception(): [ 2160.822585] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] self.force_reraise() [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] raise self.value [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] updated_port = self._update_port( [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] _ensure_no_port_binding_failure(port) [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] raise exception.PortBindingFailed(port_id=port['id']) [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] nova.exception.PortBindingFailed: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. [ 2160.822900] env[62506]: ERROR nova.compute.manager [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] [ 2160.823251] env[62506]: DEBUG nova.compute.utils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2160.824281] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 16.272s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2160.824566] env[62506]: DEBUG nova.objects.instance [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] [instance: 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62506) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 2160.827570] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Build of instance ec283c13-1c5f-4dff-9254-3c278d87935a was re-scheduled: Binding failed for port 4a72469c-bd76-46ca-b313-a0f83c357071, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2160.828131] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2160.828247] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2160.828391] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquired lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2160.828555] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2160.865081] env[62506]: ERROR nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2160.865081] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2160.865081] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2160.865081] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2160.865081] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2160.865081] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2160.865081] env[62506]: ERROR nova.compute.manager raise self.value [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2160.865081] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2160.865081] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2160.865081] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2160.865524] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2160.865524] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2160.865524] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2160.865524] env[62506]: ERROR nova.compute.manager [ 2160.865524] env[62506]: Traceback (most recent call last): [ 2160.865524] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2160.865524] env[62506]: listener.cb(fileno) [ 2160.865524] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2160.865524] env[62506]: result = function(*args, **kwargs) [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2160.865524] env[62506]: return func(*args, **kwargs) [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2160.865524] env[62506]: raise e [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2160.865524] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2160.865524] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2160.865524] env[62506]: with excutils.save_and_reraise_exception(): [ 2160.865524] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2160.865524] env[62506]: self.force_reraise() [ 2160.865524] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2160.865524] env[62506]: raise self.value [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2160.865524] env[62506]: updated_port = self._update_port( [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2160.865524] env[62506]: _ensure_no_port_binding_failure(port) [ 2160.865524] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2160.865524] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2160.866285] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2160.866285] env[62506]: Removing descriptor: 19 [ 2161.130199] env[62506]: DEBUG nova.network.neutron [-] [instance: a2735381-e738-419a-bf49-16459efbb439] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2161.151254] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2161.177473] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2161.178182] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2161.178182] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2161.178182] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2161.178392] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2161.178392] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2161.178557] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2161.178716] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2161.178892] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2161.179066] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2161.179253] env[62506]: DEBUG nova.virt.hardware [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2161.180148] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-589d45c7-4c43-48f4-95ac-081c8b08de63 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2161.188115] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e5f2d31-064c-411f-b356-621333c98190 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2161.202536] env[62506]: ERROR nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Traceback (most recent call last): [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] yield resources [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self.driver.spawn(context, instance, image_meta, [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] vm_ref = self.build_virtual_machine(instance, [ 2161.202536] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] vif_infos = vmwarevif.get_vif_info(self._session, [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] for vif in network_info: [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] return self._sync_wrapper(fn, *args, **kwargs) [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self.wait() [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self[:] = self._gt.wait() [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] return self._exit_event.wait() [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2161.202915] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] current.throw(*self._exc) [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] result = function(*args, **kwargs) [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] return func(*args, **kwargs) [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] raise e [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] nwinfo = self.network_api.allocate_for_instance( [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] created_port_ids = self._update_ports_for_instance( [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] with excutils.save_and_reraise_exception(): [ 2161.203640] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self.force_reraise() [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] raise self.value [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] updated_port = self._update_port( [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] _ensure_no_port_binding_failure(port) [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] raise exception.PortBindingFailed(port_id=port['id']) [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2161.204229] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] [ 2161.204229] env[62506]: INFO nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Terminating instance [ 2161.230415] env[62506]: DEBUG nova.network.neutron [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2161.315886] env[62506]: DEBUG nova.network.neutron [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2161.347543] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2161.424405] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2161.632684] env[62506]: INFO nova.compute.manager [-] [instance: a2735381-e738-419a-bf49-16459efbb439] Took 1.02 seconds to deallocate network for instance. [ 2161.635016] env[62506]: DEBUG nova.compute.claims [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2161.635210] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2161.707573] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2161.818827] env[62506]: DEBUG oslo_concurrency.lockutils [req-392af692-74f8-4038-bee0-db67be4a1895 req-94fab82a-1dd9-4715-8646-b6cee6050029 service nova] Releasing lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2161.819386] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquired lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2161.819664] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2161.835304] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d8ebd717-c464-4395-b9fb-a7d5f217ce43 tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.011s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2161.836460] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.157s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2161.927010] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Releasing lock "refresh_cache-ec283c13-1c5f-4dff-9254-3c278d87935a" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2161.927302] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2161.927483] env[62506]: DEBUG nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2161.927648] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2161.942530] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2162.336015] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2162.408365] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2162.445305] env[62506]: DEBUG nova.network.neutron [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2162.464867] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d205f0ac-07cf-46aa-a3a9-40c152ab7080 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.472486] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7162911e-3d20-4f93-ac10-6b2a21be8ddc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.501517] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7c16cc3-bd62-40d9-8adf-52d070b24600 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.508141] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0a5b96f-9007-4782-853f-b9cb871f91bb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.520644] env[62506]: DEBUG nova.compute.provider_tree [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2162.749069] env[62506]: DEBUG nova.compute.manager [req-c993d906-b8f0-46af-b767-0a3527a79431 req-7e7be7f5-0380-4e21-ac38-3acad5e4213e service nova] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Received event network-vif-deleted-89e25f07-3613-4320-b89e-a52b896c4a8f {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2162.910942] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Releasing lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2162.911456] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2162.911661] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2162.911976] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d16e4138-664d-4e39-ba81-235e2548574a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.921514] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2232a1a1-a826-491a-b362-132004b1426d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.943432] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance abcd5008-a127-4def-a29c-5bb8af85e196 could not be found. [ 2162.943651] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2162.943832] env[62506]: INFO nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2162.944087] env[62506]: DEBUG oslo.service.loopingcall [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2162.944334] env[62506]: DEBUG nova.compute.manager [-] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2162.944433] env[62506]: DEBUG nova.network.neutron [-] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2162.947657] env[62506]: INFO nova.compute.manager [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: ec283c13-1c5f-4dff-9254-3c278d87935a] Took 1.02 seconds to deallocate network for instance. [ 2162.959747] env[62506]: DEBUG nova.network.neutron [-] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2163.023894] env[62506]: DEBUG nova.scheduler.client.report [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2163.462714] env[62506]: DEBUG nova.network.neutron [-] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2163.528363] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.692s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2163.528993] env[62506]: ERROR nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Traceback (most recent call last): [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self.driver.spawn(context, instance, image_meta, [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] vm_ref = self.build_virtual_machine(instance, [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] vif_infos = vmwarevif.get_vif_info(self._session, [ 2163.528993] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] for vif in network_info: [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] return self._sync_wrapper(fn, *args, **kwargs) [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self.wait() [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self[:] = self._gt.wait() [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] return self._exit_event.wait() [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] current.throw(*self._exc) [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2163.529350] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] result = function(*args, **kwargs) [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] return func(*args, **kwargs) [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] raise e [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] nwinfo = self.network_api.allocate_for_instance( [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] created_port_ids = self._update_ports_for_instance( [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] with excutils.save_and_reraise_exception(): [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] self.force_reraise() [ 2163.529729] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] raise self.value [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] updated_port = self._update_port( [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] _ensure_no_port_binding_failure(port) [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] raise exception.PortBindingFailed(port_id=port['id']) [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] nova.exception.PortBindingFailed: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. [ 2163.530119] env[62506]: ERROR nova.compute.manager [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] [ 2163.530119] env[62506]: DEBUG nova.compute.utils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2163.530772] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.560s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2163.532168] env[62506]: INFO nova.compute.claims [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2163.534653] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Build of instance ed482516-f639-4f5d-8e79-cdb72c85e304 was re-scheduled: Binding failed for port c6d12d9e-9e68-433e-a3e2-3e0560c3b30f, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2163.535066] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2163.535300] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2163.535445] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2163.535607] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2163.611232] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2163.964772] env[62506]: INFO nova.compute.manager [-] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Took 1.02 seconds to deallocate network for instance. [ 2163.966894] env[62506]: DEBUG nova.compute.claims [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2163.966980] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2163.973923] env[62506]: INFO nova.scheduler.client.report [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Deleted allocations for instance ec283c13-1c5f-4dff-9254-3c278d87935a [ 2164.059172] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2164.156502] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2164.484404] env[62506]: DEBUG oslo_concurrency.lockutils [None req-f0b344c5-8fdb-4014-9874-8aaaae85631f tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "ec283c13-1c5f-4dff-9254-3c278d87935a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.512s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2164.648461] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08a0ff45-f648-4e77-8e22-601beedfcb85 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.656280] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad21afc3-12b0-4898-9ba3-7e4bc8476f62 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.659369] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-ed482516-f639-4f5d-8e79-cdb72c85e304" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2164.659580] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2164.659760] env[62506]: DEBUG nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2164.659922] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2164.688624] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2164.690154] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-495b8f80-71d7-4962-a329-ed4af9d68835 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.697858] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c88dd3d3-2b80-4427-9f74-d40477df5acb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.711243] env[62506]: DEBUG nova.compute.provider_tree [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2165.193606] env[62506]: DEBUG nova.network.neutron [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2165.214474] env[62506]: DEBUG nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2165.696694] env[62506]: INFO nova.compute.manager [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: ed482516-f639-4f5d-8e79-cdb72c85e304] Took 1.04 seconds to deallocate network for instance. [ 2165.719026] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.188s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2165.719671] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2165.722157] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 16.020s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2165.722387] env[62506]: DEBUG nova.objects.instance [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lazy-loading 'resources' on Instance uuid 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9 {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 2166.224742] env[62506]: DEBUG nova.compute.utils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2166.228462] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2166.228628] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2166.292808] env[62506]: DEBUG nova.policy [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'defd15b4b5b54251a9c79e4a9670c56b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cf42a43c67cb48ae823d00764a30d6ae', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2166.362690] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea5ff4f6-4723-427d-a925-f9cfc6603733 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2166.380641] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d7c2881-6ea3-4a14-8305-31f25e969b75 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2166.412927] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc65fc8f-4484-4555-83cd-076d6486bb40 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2166.421020] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d84956f1-a711-4604-b2cc-5174e06b3bcf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2166.433369] env[62506]: DEBUG nova.compute.provider_tree [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2166.728215] env[62506]: INFO nova.scheduler.client.report [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Deleted allocations for instance ed482516-f639-4f5d-8e79-cdb72c85e304 [ 2166.732520] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2166.767739] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Successfully created port: 0d1a549d-9c6b-468b-bd82-8fa4f255bf34 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2166.940974] env[62506]: DEBUG nova.scheduler.client.report [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2167.239279] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9cc207f4-4a1a-44e3-8911-20294bea8590 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "ed482516-f639-4f5d-8e79-cdb72c85e304" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 72.737s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2167.450789] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.728s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2167.453276] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.557s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2167.471584] env[62506]: INFO nova.scheduler.client.report [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Deleted allocations for instance 8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9 [ 2167.525085] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "5837199b-5937-4b7d-9c5b-e34c128db426" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2167.525361] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "5837199b-5937-4b7d-9c5b-e34c128db426" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2167.741671] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2167.768680] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2167.768836] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2167.769165] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2167.769165] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2167.769288] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2167.769396] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2167.769777] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2167.769777] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2167.770663] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2167.770663] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2167.770663] env[62506]: DEBUG nova.virt.hardware [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2167.771474] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16144e46-aa53-4b4b-95b0-eadedb62cd61 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2167.779535] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e486043f-e1d1-48eb-bf7d-b90725f1cf39 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2167.933674] env[62506]: DEBUG nova.compute.manager [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Received event network-changed-0d1a549d-9c6b-468b-bd82-8fa4f255bf34 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2167.933867] env[62506]: DEBUG nova.compute.manager [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Refreshing instance network info cache due to event network-changed-0d1a549d-9c6b-468b-bd82-8fa4f255bf34. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2167.934199] env[62506]: DEBUG oslo_concurrency.lockutils [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] Acquiring lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2167.934361] env[62506]: DEBUG oslo_concurrency.lockutils [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] Acquired lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2167.934545] env[62506]: DEBUG nova.network.neutron [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Refreshing network info cache for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2167.979161] env[62506]: DEBUG oslo_concurrency.lockutils [None req-60751230-9b84-4f8c-9a6e-d729a11b34ee tempest-ServerShowV254Test-527727856 tempest-ServerShowV254Test-527727856-project-member] Lock "8a1dc04e-91a3-4c5c-ab2f-05048c49f4e9" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 22.463s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2168.028508] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2168.103219] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a739fb7-ddf5-4116-8911-d1df698cc08d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2168.110671] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb083d99-1549-4a56-8754-965e44b63fc1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2168.144488] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6c8778a-bd9d-41db-af42-85da95834cad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2168.152404] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54d9e42b-ee4c-40f9-910a-ecfaad93b463 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2168.157309] env[62506]: ERROR nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2168.157309] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2168.157309] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2168.157309] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2168.157309] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2168.157309] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2168.157309] env[62506]: ERROR nova.compute.manager raise self.value [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2168.157309] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2168.157309] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2168.157309] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2168.157805] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2168.157805] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2168.157805] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2168.157805] env[62506]: ERROR nova.compute.manager [ 2168.157805] env[62506]: Traceback (most recent call last): [ 2168.157805] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2168.157805] env[62506]: listener.cb(fileno) [ 2168.157805] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2168.157805] env[62506]: result = function(*args, **kwargs) [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2168.157805] env[62506]: return func(*args, **kwargs) [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2168.157805] env[62506]: raise e [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2168.157805] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2168.157805] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2168.157805] env[62506]: with excutils.save_and_reraise_exception(): [ 2168.157805] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2168.157805] env[62506]: self.force_reraise() [ 2168.157805] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2168.157805] env[62506]: raise self.value [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2168.157805] env[62506]: updated_port = self._update_port( [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2168.157805] env[62506]: _ensure_no_port_binding_failure(port) [ 2168.157805] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2168.157805] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2168.158550] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2168.158550] env[62506]: Removing descriptor: 19 [ 2168.158550] env[62506]: ERROR nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Traceback (most recent call last): [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] yield resources [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self.driver.spawn(context, instance, image_meta, [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2168.158550] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] vm_ref = self.build_virtual_machine(instance, [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] vif_infos = vmwarevif.get_vif_info(self._session, [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] for vif in network_info: [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return self._sync_wrapper(fn, *args, **kwargs) [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self.wait() [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self[:] = self._gt.wait() [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return self._exit_event.wait() [ 2168.158879] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] result = hub.switch() [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return self.greenlet.switch() [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] result = function(*args, **kwargs) [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return func(*args, **kwargs) [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] raise e [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] nwinfo = self.network_api.allocate_for_instance( [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2168.159216] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] created_port_ids = self._update_ports_for_instance( [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] with excutils.save_and_reraise_exception(): [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self.force_reraise() [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] raise self.value [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] updated_port = self._update_port( [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] _ensure_no_port_binding_failure(port) [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2168.159586] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] raise exception.PortBindingFailed(port_id=port['id']) [ 2168.159874] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2168.159874] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] [ 2168.159874] env[62506]: INFO nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Terminating instance [ 2168.169539] env[62506]: DEBUG nova.compute.provider_tree [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2168.208418] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "45af80d9-96ec-41bd-8710-fc02c2963530" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2168.208664] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "45af80d9-96ec-41bd-8710-fc02c2963530" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2168.453027] env[62506]: DEBUG nova.network.neutron [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2168.538266] env[62506]: DEBUG nova.network.neutron [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2168.556048] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2168.662177] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquiring lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2168.672515] env[62506]: DEBUG nova.scheduler.client.report [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2168.710604] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2169.043050] env[62506]: DEBUG oslo_concurrency.lockutils [req-78b9677b-3c24-4b62-bc99-aa494aa3c5f5 req-995ac003-cf1e-470d-9c11-fdf86a1af413 service nova] Releasing lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2169.043382] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquired lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2169.043549] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2169.177326] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.724s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2169.178009] env[62506]: ERROR nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Traceback (most recent call last): [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self.driver.spawn(context, instance, image_meta, [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] vm_ref = self.build_virtual_machine(instance, [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2169.178009] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] for vif in network_info: [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] return self._sync_wrapper(fn, *args, **kwargs) [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self.wait() [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self[:] = self._gt.wait() [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] return self._exit_event.wait() [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] current.throw(*self._exc) [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2169.178310] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] result = function(*args, **kwargs) [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] return func(*args, **kwargs) [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] raise e [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] nwinfo = self.network_api.allocate_for_instance( [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] created_port_ids = self._update_ports_for_instance( [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] with excutils.save_and_reraise_exception(): [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] self.force_reraise() [ 2169.178698] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] raise self.value [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] updated_port = self._update_port( [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] _ensure_no_port_binding_failure(port) [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] raise exception.PortBindingFailed(port_id=port['id']) [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] nova.exception.PortBindingFailed: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. [ 2169.179068] env[62506]: ERROR nova.compute.manager [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] [ 2169.179068] env[62506]: DEBUG nova.compute.utils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2169.180428] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.260s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2169.183627] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Build of instance 220ccae9-6d65-4925-9fef-d555cb923fdd was re-scheduled: Binding failed for port a3bed0c8-f014-4c57-a2ee-38cfed21f155, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2169.184067] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2169.184306] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2169.184496] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquired lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2169.185040] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2169.233914] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2169.575600] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2169.720112] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2169.731818] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2169.834576] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2169.843891] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffe4774e-8e2f-4daa-ae50-4df7b88a2f11 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2169.850356] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32be979a-9575-4915-9933-c535dd6bffa3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2169.883522] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b603d2e2-5b11-4864-9df4-84e386ec483e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2169.893145] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b816975f-d82b-4fa6-abf7-66636f8e0a29 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2169.906797] env[62506]: DEBUG nova.compute.provider_tree [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2169.991338] env[62506]: DEBUG nova.compute.manager [req-1ba3d285-fc1b-4cfe-a7da-f7b23cced8f4 req-2339a642-3082-4555-9c33-96a562ac3a70 service nova] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Received event network-vif-deleted-0d1a549d-9c6b-468b-bd82-8fa4f255bf34 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2170.233284] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Releasing lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2170.233747] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2170.233950] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2170.234342] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4b6e16c5-408e-42ae-9a1c-e28d3ba3fcc7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2170.244592] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-965fe74d-d82b-4a1e-a779-fe15077143ff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2170.269481] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ba21b68c-eeae-4758-b65b-74b9780dbfcc could not be found. [ 2170.269603] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2170.269783] env[62506]: INFO nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2170.270032] env[62506]: DEBUG oslo.service.loopingcall [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2170.270257] env[62506]: DEBUG nova.compute.manager [-] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2170.270352] env[62506]: DEBUG nova.network.neutron [-] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2170.286109] env[62506]: DEBUG nova.network.neutron [-] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2170.339306] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Releasing lock "refresh_cache-220ccae9-6d65-4925-9fef-d555cb923fdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2170.339552] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2170.339737] env[62506]: DEBUG nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2170.339898] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2170.354491] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2170.409795] env[62506]: DEBUG nova.scheduler.client.report [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2170.610227] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2170.610470] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2170.788958] env[62506]: DEBUG nova.network.neutron [-] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2170.857161] env[62506]: DEBUG nova.network.neutron [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2170.915033] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.734s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2170.916819] env[62506]: ERROR nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Traceback (most recent call last): [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self.driver.spawn(context, instance, image_meta, [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] vm_ref = self.build_virtual_machine(instance, [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] vif_infos = vmwarevif.get_vif_info(self._session, [ 2170.916819] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] for vif in network_info: [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] return self._sync_wrapper(fn, *args, **kwargs) [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self.wait() [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self[:] = self._gt.wait() [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] return self._exit_event.wait() [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] current.throw(*self._exc) [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2170.917244] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] result = function(*args, **kwargs) [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] return func(*args, **kwargs) [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] raise e [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] nwinfo = self.network_api.allocate_for_instance( [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] created_port_ids = self._update_ports_for_instance( [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] with excutils.save_and_reraise_exception(): [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] self.force_reraise() [ 2170.917605] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] raise self.value [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] updated_port = self._update_port( [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] _ensure_no_port_binding_failure(port) [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] raise exception.PortBindingFailed(port_id=port['id']) [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] nova.exception.PortBindingFailed: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. [ 2170.917927] env[62506]: ERROR nova.compute.manager [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] [ 2170.917927] env[62506]: DEBUG nova.compute.utils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2170.919939] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.475s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2170.922995] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Build of instance ed4f2610-e5dd-4c56-bcc0-71ec198f4833 was re-scheduled: Binding failed for port f08de356-f2ca-43cc-99c0-160d2cea8f2d, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2170.923525] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2170.923801] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2170.923992] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2170.924223] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2171.113637] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2171.292703] env[62506]: INFO nova.compute.manager [-] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Took 1.02 seconds to deallocate network for instance. [ 2171.295730] env[62506]: DEBUG nova.compute.claims [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2171.295935] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2171.359955] env[62506]: INFO nova.compute.manager [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 220ccae9-6d65-4925-9fef-d555cb923fdd] Took 1.02 seconds to deallocate network for instance. [ 2171.451627] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2171.533178] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2171.580304] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e642dc0e-9502-498e-a0df-50436e6e881d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2171.588040] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e8736f2-bf2d-4358-8ae1-05ea0b88998f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2171.618964] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f720f150-9778-46fa-8cca-d7798101ed2a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2171.625830] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95b65032-5a3b-4ffd-a4b9-af789a78ebe9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2171.640579] env[62506]: DEBUG nova.compute.provider_tree [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2172.036020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-ed4f2610-e5dd-4c56-bcc0-71ec198f4833" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2172.036301] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2172.036490] env[62506]: DEBUG nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2172.036656] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2172.051515] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2172.143543] env[62506]: DEBUG nova.scheduler.client.report [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2172.189231] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "5f77a974-8a53-439f-8700-736de9dba94e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2172.189462] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "5f77a974-8a53-439f-8700-736de9dba94e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2172.399651] env[62506]: INFO nova.scheduler.client.report [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Deleted allocations for instance 220ccae9-6d65-4925-9fef-d555cb923fdd [ 2172.554229] env[62506]: DEBUG nova.network.neutron [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2172.648529] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.730s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2172.649200] env[62506]: ERROR nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Traceback (most recent call last): [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self.driver.spawn(context, instance, image_meta, [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] vm_ref = self.build_virtual_machine(instance, [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] vif_infos = vmwarevif.get_vif_info(self._session, [ 2172.649200] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] for vif in network_info: [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] return self._sync_wrapper(fn, *args, **kwargs) [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self.wait() [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self[:] = self._gt.wait() [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] return self._exit_event.wait() [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] current.throw(*self._exc) [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2172.649568] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] result = function(*args, **kwargs) [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] return func(*args, **kwargs) [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] raise e [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] nwinfo = self.network_api.allocate_for_instance( [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] created_port_ids = self._update_ports_for_instance( [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] with excutils.save_and_reraise_exception(): [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] self.force_reraise() [ 2172.649944] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] raise self.value [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] updated_port = self._update_port( [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] _ensure_no_port_binding_failure(port) [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] raise exception.PortBindingFailed(port_id=port['id']) [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] nova.exception.PortBindingFailed: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. [ 2172.650347] env[62506]: ERROR nova.compute.manager [instance: 7d013470-bf62-4db4-84f0-96179b417abf] [ 2172.650347] env[62506]: DEBUG nova.compute.utils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2172.651155] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.107s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2172.654262] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Build of instance 7d013470-bf62-4db4-84f0-96179b417abf was re-scheduled: Binding failed for port ed27e006-061c-4877-aef2-24bf727d77c4, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2172.654604] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2172.654827] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2172.654975] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2172.655148] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2172.691400] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2172.907227] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2c754c1c-8d83-4cd6-b7fe-8c90664c0390 tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "220ccae9-6d65-4925-9fef-d555cb923fdd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 54.209s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2173.059301] env[62506]: INFO nova.compute.manager [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ed4f2610-e5dd-4c56-bcc0-71ec198f4833] Took 1.02 seconds to deallocate network for instance. [ 2173.181084] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2173.211413] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2173.264292] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2173.300338] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a45fefca-e920-47f5-b6b2-7fdb09724214 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2173.311396] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-162645db-0e9a-43c0-b3b4-3cbddf27ad59 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2173.340344] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95762acc-5273-4227-a60f-3dbf7d698de0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2173.348519] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a25a06c-311c-4b03-9bdc-1f7500f3384f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2173.362110] env[62506]: DEBUG nova.compute.provider_tree [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2173.772324] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-7d013470-bf62-4db4-84f0-96179b417abf" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2173.772324] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2173.772324] env[62506]: DEBUG nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2173.772324] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2173.788302] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2173.866333] env[62506]: DEBUG nova.scheduler.client.report [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2174.094672] env[62506]: INFO nova.scheduler.client.report [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Deleted allocations for instance ed4f2610-e5dd-4c56-bcc0-71ec198f4833 [ 2174.291155] env[62506]: DEBUG nova.network.neutron [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2174.371411] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.720s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2174.372123] env[62506]: ERROR nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Traceback (most recent call last): [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self.driver.spawn(context, instance, image_meta, [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] vm_ref = self.build_virtual_machine(instance, [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] vif_infos = vmwarevif.get_vif_info(self._session, [ 2174.372123] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] for vif in network_info: [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] return self._sync_wrapper(fn, *args, **kwargs) [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self.wait() [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self[:] = self._gt.wait() [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] return self._exit_event.wait() [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] current.throw(*self._exc) [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2174.372496] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] result = function(*args, **kwargs) [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] return func(*args, **kwargs) [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] raise e [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] nwinfo = self.network_api.allocate_for_instance( [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] created_port_ids = self._update_ports_for_instance( [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] with excutils.save_and_reraise_exception(): [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] self.force_reraise() [ 2174.372803] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] raise self.value [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] updated_port = self._update_port( [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] _ensure_no_port_binding_failure(port) [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] raise exception.PortBindingFailed(port_id=port['id']) [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] nova.exception.PortBindingFailed: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. [ 2174.373169] env[62506]: ERROR nova.compute.manager [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] [ 2174.373169] env[62506]: DEBUG nova.compute.utils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2174.374251] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.739s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2174.377351] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Build of instance 99973068-d1a5-4809-b722-b6dd24ad3a12 was re-scheduled: Binding failed for port f903a622-c2b3-4b0e-8f69-fb0c339ec506, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2174.378200] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2174.378306] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquiring lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2174.378512] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Acquired lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2174.379489] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2174.609133] env[62506]: DEBUG oslo_concurrency.lockutils [None req-b31e5050-b067-4904-873f-dc0ea8743ec7 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "ed4f2610-e5dd-4c56-bcc0-71ec198f4833" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 47.796s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2174.797023] env[62506]: INFO nova.compute.manager [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 7d013470-bf62-4db4-84f0-96179b417abf] Took 1.02 seconds to deallocate network for instance. [ 2174.899824] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "852705fb-d732-4c79-ba20-58f04806f15c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2174.900080] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "852705fb-d732-4c79-ba20-58f04806f15c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2174.902684] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2175.013458] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2175.016477] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5dd066cb-1114-4d25-aeba-20b91f739d9d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2175.024970] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d8eb96e-fad1-4ca1-8cd7-f8ab534aa80c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2175.055039] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a00fee0-eee8-472b-9243-377273c2e10f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2175.062711] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38ce7e13-ac13-4953-a6c7-207580ec3f35 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2175.076200] env[62506]: DEBUG nova.compute.provider_tree [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2175.405333] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2175.517123] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Releasing lock "refresh_cache-99973068-d1a5-4809-b722-b6dd24ad3a12" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2175.517375] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2175.517560] env[62506]: DEBUG nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2175.517725] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2175.534086] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2175.579210] env[62506]: DEBUG nova.scheduler.client.report [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2175.826259] env[62506]: INFO nova.scheduler.client.report [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Deleted allocations for instance 7d013470-bf62-4db4-84f0-96179b417abf [ 2175.923209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2175.953009] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "9e5fbbf6-2ace-4193-8add-5ae16c79e540" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2175.953333] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "9e5fbbf6-2ace-4193-8add-5ae16c79e540" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2176.037987] env[62506]: DEBUG nova.network.neutron [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2176.084042] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.710s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2176.084759] env[62506]: ERROR nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] Traceback (most recent call last): [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self.driver.spawn(context, instance, image_meta, [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] vm_ref = self.build_virtual_machine(instance, [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] vif_infos = vmwarevif.get_vif_info(self._session, [ 2176.084759] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] for vif in network_info: [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] return self._sync_wrapper(fn, *args, **kwargs) [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self.wait() [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self[:] = self._gt.wait() [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] return self._exit_event.wait() [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] current.throw(*self._exc) [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2176.085139] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] result = function(*args, **kwargs) [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] return func(*args, **kwargs) [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] raise e [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] nwinfo = self.network_api.allocate_for_instance( [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] created_port_ids = self._update_ports_for_instance( [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] with excutils.save_and_reraise_exception(): [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] self.force_reraise() [ 2176.085516] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] raise self.value [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] updated_port = self._update_port( [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] _ensure_no_port_binding_failure(port) [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] raise exception.PortBindingFailed(port_id=port['id']) [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] nova.exception.PortBindingFailed: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. [ 2176.085874] env[62506]: ERROR nova.compute.manager [instance: a2735381-e738-419a-bf49-16459efbb439] [ 2176.085874] env[62506]: DEBUG nova.compute.utils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2176.086756] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.120s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2176.089859] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Build of instance a2735381-e738-419a-bf49-16459efbb439 was re-scheduled: Binding failed for port 53318629-e4cd-4143-abb7-2a7d0c776bf6, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2176.089859] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2176.090090] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquiring lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2176.090239] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Acquired lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2176.090446] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2176.335028] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3fbd7273-d3b5-4a8e-885c-ffa4bfa57e39 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "7d013470-bf62-4db4-84f0-96179b417abf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 40.198s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2176.455748] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2176.539832] env[62506]: INFO nova.compute.manager [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] [instance: 99973068-d1a5-4809-b722-b6dd24ad3a12] Took 1.02 seconds to deallocate network for instance. [ 2176.612478] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2176.696251] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2176.700469] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc515ec6-f9b6-4ebe-864e-49abd4c6ebbb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2176.708439] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10922d4e-a536-4e7d-843e-012a19ba5e43 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2176.739415] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72ac2164-4e51-47dc-a0cb-711e986a71ee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2176.746749] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee0a75b7-795e-42c7-a9be-f53e52e74840 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2176.760365] env[62506]: DEBUG nova.compute.provider_tree [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2176.980272] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2177.199148] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Releasing lock "refresh_cache-a2735381-e738-419a-bf49-16459efbb439" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2177.199418] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2177.199600] env[62506]: DEBUG nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2177.199772] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2177.216704] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2177.264627] env[62506]: DEBUG nova.scheduler.client.report [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2177.567872] env[62506]: INFO nova.scheduler.client.report [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Deleted allocations for instance 99973068-d1a5-4809-b722-b6dd24ad3a12 [ 2177.719319] env[62506]: DEBUG nova.network.neutron [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2177.729041] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "dd2b0afa-7224-4d7e-8ff2-4142abcdef10" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2177.729293] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "dd2b0afa-7224-4d7e-8ff2-4142abcdef10" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2177.769540] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.683s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2177.770180] env[62506]: ERROR nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Traceback (most recent call last): [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self.driver.spawn(context, instance, image_meta, [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] vm_ref = self.build_virtual_machine(instance, [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] vif_infos = vmwarevif.get_vif_info(self._session, [ 2177.770180] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] for vif in network_info: [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] return self._sync_wrapper(fn, *args, **kwargs) [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self.wait() [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self[:] = self._gt.wait() [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] return self._exit_event.wait() [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] current.throw(*self._exc) [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2177.770564] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] result = function(*args, **kwargs) [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] return func(*args, **kwargs) [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] raise e [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] nwinfo = self.network_api.allocate_for_instance( [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] created_port_ids = self._update_ports_for_instance( [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] with excutils.save_and_reraise_exception(): [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] self.force_reraise() [ 2177.770923] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] raise self.value [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] updated_port = self._update_port( [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] _ensure_no_port_binding_failure(port) [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] raise exception.PortBindingFailed(port_id=port['id']) [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] nova.exception.PortBindingFailed: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. [ 2177.771259] env[62506]: ERROR nova.compute.manager [instance: abcd5008-a127-4def-a29c-5bb8af85e196] [ 2177.771259] env[62506]: DEBUG nova.compute.utils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2177.772007] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.216s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2177.773445] env[62506]: INFO nova.compute.claims [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2177.776187] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Build of instance abcd5008-a127-4def-a29c-5bb8af85e196 was re-scheduled: Binding failed for port 89e25f07-3613-4320-b89e-a52b896c4a8f, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2177.776549] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2177.776759] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2177.776948] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquired lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2177.777093] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2178.075262] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3654249e-fc5e-431f-8149-ff58e3078135 tempest-ServersTestBootFromVolume-778849418 tempest-ServersTestBootFromVolume-778849418-project-member] Lock "99973068-d1a5-4809-b722-b6dd24ad3a12" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 47.522s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2178.222594] env[62506]: INFO nova.compute.manager [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] [instance: a2735381-e738-419a-bf49-16459efbb439] Took 1.02 seconds to deallocate network for instance. [ 2178.231343] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2178.296810] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2178.376817] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2178.756145] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2178.881128] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Releasing lock "refresh_cache-abcd5008-a127-4def-a29c-5bb8af85e196" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2178.881378] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2178.881561] env[62506]: DEBUG nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2178.881725] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2178.904143] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2178.905159] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4a4cb2b-2747-4f9c-9d97-ae33045a8f65 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2178.913761] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47f08659-16f3-45a8-b222-92a2b1f77866 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2178.944640] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-208a39d1-5a33-48f9-82a7-275e31ed4601 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2178.951950] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abea5bfe-7fd8-44a0-a56c-2883f9afd2f6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2178.967342] env[62506]: DEBUG nova.compute.provider_tree [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2179.255832] env[62506]: INFO nova.scheduler.client.report [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Deleted allocations for instance a2735381-e738-419a-bf49-16459efbb439 [ 2179.408776] env[62506]: DEBUG nova.network.neutron [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2179.470850] env[62506]: DEBUG nova.scheduler.client.report [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2179.763434] env[62506]: DEBUG oslo_concurrency.lockutils [None req-d511bed4-1859-41f0-85cc-92fec0f5bdbe tempest-AttachInterfacesTestJSON-1678676510 tempest-AttachInterfacesTestJSON-1678676510-project-member] Lock "a2735381-e738-419a-bf49-16459efbb439" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 40.867s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2179.912504] env[62506]: INFO nova.compute.manager [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: abcd5008-a127-4def-a29c-5bb8af85e196] Took 1.03 seconds to deallocate network for instance. [ 2179.975738] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2179.976160] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2179.979178] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.745s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2179.980545] env[62506]: INFO nova.compute.claims [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2180.484315] env[62506]: DEBUG nova.compute.utils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2180.487709] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2180.487709] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2180.566958] env[62506]: DEBUG nova.policy [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4d5b44f1856745d6bdca5ed7829d1a44', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bd95448919b446be92ee08ed1e17a8a1', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2180.953080] env[62506]: INFO nova.scheduler.client.report [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Deleted allocations for instance abcd5008-a127-4def-a29c-5bb8af85e196 [ 2180.997020] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2181.073030] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Successfully created port: 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2181.098870] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a69d8c9a-1b83-46d6-8972-0ef9a6718eb1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2181.108837] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c14e6d2-605d-437b-8af0-29cce5632abc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2181.897214] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9f23ce55-475e-402c-b5a4-a4924bbb61f6 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "abcd5008-a127-4def-a29c-5bb8af85e196" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 42.690s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2181.904297] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Successfully created port: 29785532-268e-4e1d-bf1d-2c46dd2cd625 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2181.904691] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad53a0e8-9d2c-416d-944d-a8ea9ed516bc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2181.912889] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68e0f174-4d3d-4a2b-ab99-26f207d34605 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2181.926977] env[62506]: DEBUG nova.compute.provider_tree [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2182.402368] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2182.430333] env[62506]: DEBUG nova.scheduler.client.report [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2182.435999] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2182.436700] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2182.436845] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2182.437411] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2182.437411] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2182.437519] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2182.437985] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2182.437985] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2182.437985] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2182.438166] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2182.438345] env[62506]: DEBUG nova.virt.hardware [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2182.439184] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8551a0d6-d6e6-4f74-9811-aae6a6afee70 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2182.448292] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-396b35f0-863d-4a6a-898b-0120c3e11bc5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2182.943739] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.965s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2182.944317] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2182.948795] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 11.835s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2182.948795] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2182.948924] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2182.949790] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.653s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2182.953645] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-753b8ece-6f46-4897-906a-7b4825ee4793 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2182.962717] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68642da2-72ba-4bf7-bf95-f2e0993f96cb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2182.985374] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71018fce-3e13-44ea-a8cf-6e5a158890a5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2182.992808] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8e2e64e-2cfe-4972-ac17-e3ef299095c2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2183.027192] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181246MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2183.027192] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2183.368460] env[62506]: ERROR nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2183.368460] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2183.368460] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2183.368460] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2183.368460] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2183.368460] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2183.368460] env[62506]: ERROR nova.compute.manager raise self.value [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2183.368460] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2183.368460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2183.368460] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2183.369505] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2183.369505] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2183.369505] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2183.369505] env[62506]: ERROR nova.compute.manager [ 2183.369505] env[62506]: Traceback (most recent call last): [ 2183.369505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2183.369505] env[62506]: listener.cb(fileno) [ 2183.369505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2183.369505] env[62506]: result = function(*args, **kwargs) [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2183.369505] env[62506]: return func(*args, **kwargs) [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2183.369505] env[62506]: raise e [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2183.369505] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2183.369505] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2183.369505] env[62506]: with excutils.save_and_reraise_exception(): [ 2183.369505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2183.369505] env[62506]: self.force_reraise() [ 2183.369505] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2183.369505] env[62506]: raise self.value [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2183.369505] env[62506]: updated_port = self._update_port( [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2183.369505] env[62506]: _ensure_no_port_binding_failure(port) [ 2183.369505] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2183.369505] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2183.370966] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2183.370966] env[62506]: Removing descriptor: 20 [ 2183.370966] env[62506]: ERROR nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Traceback (most recent call last): [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] yield resources [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self.driver.spawn(context, instance, image_meta, [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2183.370966] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] vm_ref = self.build_virtual_machine(instance, [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] vif_infos = vmwarevif.get_vif_info(self._session, [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] for vif in network_info: [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return self._sync_wrapper(fn, *args, **kwargs) [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self.wait() [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self[:] = self._gt.wait() [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return self._exit_event.wait() [ 2183.371576] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] result = hub.switch() [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return self.greenlet.switch() [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] result = function(*args, **kwargs) [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return func(*args, **kwargs) [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] raise e [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] nwinfo = self.network_api.allocate_for_instance( [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2183.372088] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] created_port_ids = self._update_ports_for_instance( [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] with excutils.save_and_reraise_exception(): [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self.force_reraise() [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] raise self.value [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] updated_port = self._update_port( [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] _ensure_no_port_binding_failure(port) [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2183.372530] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] raise exception.PortBindingFailed(port_id=port['id']) [ 2183.372977] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2183.372977] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] [ 2183.372977] env[62506]: INFO nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Terminating instance [ 2183.457025] env[62506]: DEBUG nova.compute.utils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2183.457025] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2183.457025] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2183.508729] env[62506]: DEBUG nova.policy [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ab67b57f49ad47ea9aab58b4f7bc563f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c97e56471ed471aa4a0812ec92e082c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2183.514459] env[62506]: DEBUG nova.compute.manager [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Received event network-changed-48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2183.514648] env[62506]: DEBUG nova.compute.manager [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Refreshing instance network info cache due to event network-changed-48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2183.515381] env[62506]: DEBUG oslo_concurrency.lockutils [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] Acquiring lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2183.515451] env[62506]: DEBUG oslo_concurrency.lockutils [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] Acquired lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2183.516267] env[62506]: DEBUG nova.network.neutron [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Refreshing network info cache for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2183.592864] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a77ec8d5-a2ce-49f9-91ce-7be6c96b783d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2183.601287] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bffcf44-08d9-4ddb-b8dc-1626268b940a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2183.632850] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0217ea9d-f432-4986-bea6-0684ea6b5bb6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2183.641461] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75c05a3e-32d0-4284-8c13-b7adc8ce38ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2183.658382] env[62506]: DEBUG nova.compute.provider_tree [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2183.830314] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Successfully created port: b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2183.875193] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2183.966596] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2184.075513] env[62506]: DEBUG nova.network.neutron [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2184.188150] env[62506]: ERROR nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [req-3f629778-0670-464f-87e3-f5cd34b41fe3] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-3f629778-0670-464f-87e3-f5cd34b41fe3"}]}: nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2184.208751] env[62506]: DEBUG nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 2184.226077] env[62506]: DEBUG nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 2184.226330] env[62506]: DEBUG nova.compute.provider_tree [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2184.256184] env[62506]: DEBUG nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 2184.264876] env[62506]: DEBUG nova.network.neutron [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2184.278158] env[62506]: DEBUG nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 2184.414189] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e0c0045-231a-4a61-8bce-63cd6d260995 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.422740] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb084bc5-1fa4-4255-935f-3367d463e2a1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.452916] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-289fe78d-bce6-40ec-846c-cf72fb92a7aa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.465248] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b40f943b-0998-400d-b406-0bef4bbac211 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.487821] env[62506]: DEBUG nova.compute.provider_tree [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2184.761970] env[62506]: ERROR nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2184.761970] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2184.761970] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2184.761970] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2184.761970] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2184.761970] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2184.761970] env[62506]: ERROR nova.compute.manager raise self.value [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2184.761970] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2184.761970] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2184.761970] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2184.762598] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2184.762598] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2184.762598] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2184.762598] env[62506]: ERROR nova.compute.manager [ 2184.762598] env[62506]: Traceback (most recent call last): [ 2184.762598] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2184.762598] env[62506]: listener.cb(fileno) [ 2184.762598] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2184.762598] env[62506]: result = function(*args, **kwargs) [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2184.762598] env[62506]: return func(*args, **kwargs) [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2184.762598] env[62506]: raise e [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2184.762598] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2184.762598] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2184.762598] env[62506]: with excutils.save_and_reraise_exception(): [ 2184.762598] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2184.762598] env[62506]: self.force_reraise() [ 2184.762598] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2184.762598] env[62506]: raise self.value [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2184.762598] env[62506]: updated_port = self._update_port( [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2184.762598] env[62506]: _ensure_no_port_binding_failure(port) [ 2184.762598] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2184.762598] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2184.763430] env[62506]: nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2184.763430] env[62506]: Removing descriptor: 20 [ 2184.767579] env[62506]: DEBUG oslo_concurrency.lockutils [req-f88aca02-dbf1-48f6-aff0-5d38ae3fa64d req-53a2725c-53ec-43f7-a886-4ab7108b9292 service nova] Releasing lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2184.767887] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquired lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2184.768095] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2184.990636] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2185.014018] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2185.014018] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2185.014018] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2185.014313] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2185.014576] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2185.014907] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2185.015280] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2185.017062] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2185.017062] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2185.017062] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2185.017062] env[62506]: DEBUG nova.virt.hardware [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2185.017062] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30f1d0e4-87c5-4d91-9e07-6adb1362c884 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2185.025382] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-347ada1c-4cd3-47e5-87bc-0600920ccb01 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2185.030328] env[62506]: DEBUG nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 104 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 2185.030328] env[62506]: DEBUG nova.compute.provider_tree [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 104 to 105 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 2185.030528] env[62506]: DEBUG nova.compute.provider_tree [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2185.046424] env[62506]: ERROR nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Traceback (most recent call last): [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] yield resources [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self.driver.spawn(context, instance, image_meta, [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] vm_ref = self.build_virtual_machine(instance, [ 2185.046424] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] vif_infos = vmwarevif.get_vif_info(self._session, [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] for vif in network_info: [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] return self._sync_wrapper(fn, *args, **kwargs) [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self.wait() [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self[:] = self._gt.wait() [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] return self._exit_event.wait() [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2185.046812] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] current.throw(*self._exc) [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] result = function(*args, **kwargs) [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] return func(*args, **kwargs) [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] raise e [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] nwinfo = self.network_api.allocate_for_instance( [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] created_port_ids = self._update_ports_for_instance( [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] with excutils.save_and_reraise_exception(): [ 2185.047143] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self.force_reraise() [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] raise self.value [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] updated_port = self._update_port( [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] _ensure_no_port_binding_failure(port) [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] raise exception.PortBindingFailed(port_id=port['id']) [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2185.047464] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] [ 2185.047464] env[62506]: INFO nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Terminating instance [ 2185.303035] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2185.394365] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2185.537013] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.587s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2185.537539] env[62506]: ERROR nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Traceback (most recent call last): [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self.driver.spawn(context, instance, image_meta, [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] vm_ref = self.build_virtual_machine(instance, [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] vif_infos = vmwarevif.get_vif_info(self._session, [ 2185.537539] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] for vif in network_info: [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return self._sync_wrapper(fn, *args, **kwargs) [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self.wait() [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self[:] = self._gt.wait() [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return self._exit_event.wait() [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] result = hub.switch() [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2185.537837] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return self.greenlet.switch() [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] result = function(*args, **kwargs) [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] return func(*args, **kwargs) [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] raise e [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] nwinfo = self.network_api.allocate_for_instance( [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] created_port_ids = self._update_ports_for_instance( [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] with excutils.save_and_reraise_exception(): [ 2185.538281] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] self.force_reraise() [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] raise self.value [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] updated_port = self._update_port( [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] _ensure_no_port_binding_failure(port) [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] raise exception.PortBindingFailed(port_id=port['id']) [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] nova.exception.PortBindingFailed: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. [ 2185.538692] env[62506]: ERROR nova.compute.manager [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] [ 2185.539035] env[62506]: DEBUG nova.compute.utils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2185.540767] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.329s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2185.542361] env[62506]: INFO nova.compute.claims [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2185.545963] env[62506]: DEBUG nova.compute.manager [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Received event network-vif-deleted-48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2185.546174] env[62506]: DEBUG nova.compute.manager [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Received event network-changed-b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2185.546341] env[62506]: DEBUG nova.compute.manager [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Refreshing instance network info cache due to event network-changed-b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2185.546553] env[62506]: DEBUG oslo_concurrency.lockutils [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] Acquiring lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2185.546695] env[62506]: DEBUG oslo_concurrency.lockutils [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] Acquired lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2185.546853] env[62506]: DEBUG nova.network.neutron [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Refreshing network info cache for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2185.548136] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Build of instance ba21b68c-eeae-4758-b65b-74b9780dbfcc was re-scheduled: Binding failed for port 0d1a549d-9c6b-468b-bd82-8fa4f255bf34, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2185.548591] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2185.548784] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquiring lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2185.548931] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Acquired lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2185.549180] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2185.551694] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2185.596118] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "26472ec8-3e42-42cf-b841-9d976f67afdd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2185.596345] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "26472ec8-3e42-42cf-b841-9d976f67afdd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2185.897187] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Releasing lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2185.897655] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2185.897851] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2185.898183] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-08ba498d-4981-4a4d-93e8-1dff0df0e0bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2185.908099] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e338704-910f-49e2-97ef-5e160874cc7b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2185.933524] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5837199b-5937-4b7d-9c5b-e34c128db426 could not be found. [ 2185.933524] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2185.933524] env[62506]: INFO nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2185.933524] env[62506]: DEBUG oslo.service.loopingcall [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2185.933524] env[62506]: DEBUG nova.compute.manager [-] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2185.933524] env[62506]: DEBUG nova.network.neutron [-] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2185.976798] env[62506]: DEBUG nova.network.neutron [-] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2185.992808] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "84586d70-87bf-45a1-9f04-7ecfca539504" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2185.993381] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "84586d70-87bf-45a1-9f04-7ecfca539504" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2186.025525] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "e6a3f17c-e342-4445-8734-75e18b29b482" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2186.026040] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "e6a3f17c-e342-4445-8734-75e18b29b482" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2186.058663] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "118f53cf-a170-44d3-bacd-9c94e98d8c5d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2186.058910] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "118f53cf-a170-44d3-bacd-9c94e98d8c5d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2186.082146] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2186.085712] env[62506]: DEBUG nova.network.neutron [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2186.098032] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2186.245770] env[62506]: DEBUG nova.network.neutron [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2186.253732] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2186.495710] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2186.528018] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2186.615902] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2186.681876] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fe39072-8099-4e2b-8e39-493f7c2577ea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2186.689416] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34cee592-97fd-490e-9fe0-1183f377c00d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2186.719854] env[62506]: DEBUG nova.network.neutron [-] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2186.721547] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-491852e2-32ee-49ad-aa94-8312a95cfa95 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2186.728784] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e998415a-3b1b-4580-9d90-f57bd2d2348e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2186.742191] env[62506]: DEBUG nova.compute.provider_tree [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2186.748296] env[62506]: DEBUG oslo_concurrency.lockutils [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] Releasing lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2186.748482] env[62506]: DEBUG nova.compute.manager [req-98104a56-9b05-45ca-9bf3-1813b7f24493 req-c0b91c43-a1fe-4b94-b6d7-0c8d83c6a462 service nova] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Received event network-vif-deleted-b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2186.748809] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2186.748978] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2186.755585] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Releasing lock "refresh_cache-ba21b68c-eeae-4758-b65b-74b9780dbfcc" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2186.755754] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2186.755931] env[62506]: DEBUG nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2186.756105] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2186.771215] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2187.013256] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2187.044954] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2187.225020] env[62506]: INFO nova.compute.manager [-] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Took 1.29 seconds to deallocate network for instance. [ 2187.227436] env[62506]: DEBUG nova.compute.claims [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2187.227436] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2187.246130] env[62506]: DEBUG nova.scheduler.client.report [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2187.265657] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2187.273018] env[62506]: DEBUG nova.network.neutron [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2187.362510] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2187.750492] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.210s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2187.750996] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2187.753945] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.831s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2187.755218] env[62506]: INFO nova.compute.claims [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2187.775332] env[62506]: INFO nova.compute.manager [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] [instance: ba21b68c-eeae-4758-b65b-74b9780dbfcc] Took 1.02 seconds to deallocate network for instance. [ 2187.865460] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2187.865886] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2187.866093] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2187.866430] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9d965d31-6df3-4965-83f6-122dda3ea97e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2187.875544] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-faf40ec4-15e1-4369-bdbb-2c2c55a74ffd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2187.896956] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 45af80d9-96ec-41bd-8710-fc02c2963530 could not be found. [ 2187.897200] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2187.897454] env[62506]: INFO nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2187.897625] env[62506]: DEBUG oslo.service.loopingcall [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2187.897840] env[62506]: DEBUG nova.compute.manager [-] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2187.897935] env[62506]: DEBUG nova.network.neutron [-] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2187.923886] env[62506]: DEBUG nova.network.neutron [-] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2188.260203] env[62506]: DEBUG nova.compute.utils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2188.264544] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2188.265192] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2188.314033] env[62506]: DEBUG nova.policy [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bbccae76c293437b84873cc656a055c3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a983195f14a44160a2cf5c170bd79ffd', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2188.426854] env[62506]: DEBUG nova.network.neutron [-] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2188.645364] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Successfully created port: d55540bb-f511-4fea-b6f2-93ba84855be0 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2188.769241] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2188.805144] env[62506]: INFO nova.scheduler.client.report [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Deleted allocations for instance ba21b68c-eeae-4758-b65b-74b9780dbfcc [ 2188.910434] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d4ab7b5-3e64-475f-9e28-02486a8a6e15 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2188.917677] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc31f722-90d1-4806-8eb1-05675fc5e547 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2188.947286] env[62506]: INFO nova.compute.manager [-] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Took 1.05 seconds to deallocate network for instance. [ 2188.949865] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a71c54d5-50d3-40e5-9096-b9aef59ebc7f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2188.953019] env[62506]: DEBUG nova.compute.claims [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2188.953112] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2188.958532] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff98decc-eb54-4db0-9d47-b5477160c93d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2188.973509] env[62506]: DEBUG nova.compute.provider_tree [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2189.315079] env[62506]: DEBUG oslo_concurrency.lockutils [None req-afb7a21f-18eb-4386-9b3e-079f219239a9 tempest-ServerAddressesTestJSON-606726954 tempest-ServerAddressesTestJSON-606726954-project-member] Lock "ba21b68c-eeae-4758-b65b-74b9780dbfcc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 42.390s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2189.347413] env[62506]: DEBUG nova.compute.manager [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Received event network-changed-d55540bb-f511-4fea-b6f2-93ba84855be0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2189.347644] env[62506]: DEBUG nova.compute.manager [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Refreshing instance network info cache due to event network-changed-d55540bb-f511-4fea-b6f2-93ba84855be0. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2189.347972] env[62506]: DEBUG oslo_concurrency.lockutils [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] Acquiring lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2189.348032] env[62506]: DEBUG oslo_concurrency.lockutils [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] Acquired lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2189.348193] env[62506]: DEBUG nova.network.neutron [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Refreshing network info cache for port d55540bb-f511-4fea-b6f2-93ba84855be0 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2189.476457] env[62506]: DEBUG nova.scheduler.client.report [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2189.549464] env[62506]: ERROR nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2189.549464] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2189.549464] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2189.549464] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2189.549464] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2189.549464] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2189.549464] env[62506]: ERROR nova.compute.manager raise self.value [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2189.549464] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2189.549464] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2189.549464] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2189.549952] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2189.549952] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2189.549952] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2189.549952] env[62506]: ERROR nova.compute.manager [ 2189.549952] env[62506]: Traceback (most recent call last): [ 2189.549952] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2189.549952] env[62506]: listener.cb(fileno) [ 2189.549952] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2189.549952] env[62506]: result = function(*args, **kwargs) [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2189.549952] env[62506]: return func(*args, **kwargs) [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2189.549952] env[62506]: raise e [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2189.549952] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2189.549952] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2189.549952] env[62506]: with excutils.save_and_reraise_exception(): [ 2189.549952] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2189.549952] env[62506]: self.force_reraise() [ 2189.549952] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2189.549952] env[62506]: raise self.value [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2189.549952] env[62506]: updated_port = self._update_port( [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2189.549952] env[62506]: _ensure_no_port_binding_failure(port) [ 2189.549952] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2189.549952] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2189.550791] env[62506]: nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2189.550791] env[62506]: Removing descriptor: 15 [ 2189.778471] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2189.811506] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2189.811790] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2189.811971] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2189.812249] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2189.812438] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2189.812625] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2189.812903] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2189.813181] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2189.813373] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2189.813640] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2189.813835] env[62506]: DEBUG nova.virt.hardware [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2189.814863] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f5242d2-78b5-4656-9a28-73b66927467e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2189.818271] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2189.827485] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-601750a1-8240-4c0c-bbba-680d7e9dc2e3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2189.842966] env[62506]: ERROR nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Traceback (most recent call last): [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] yield resources [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self.driver.spawn(context, instance, image_meta, [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] vm_ref = self.build_virtual_machine(instance, [ 2189.842966] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] vif_infos = vmwarevif.get_vif_info(self._session, [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] for vif in network_info: [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] return self._sync_wrapper(fn, *args, **kwargs) [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self.wait() [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self[:] = self._gt.wait() [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] return self._exit_event.wait() [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2189.843415] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] current.throw(*self._exc) [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] result = function(*args, **kwargs) [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] return func(*args, **kwargs) [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] raise e [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] nwinfo = self.network_api.allocate_for_instance( [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] created_port_ids = self._update_ports_for_instance( [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] with excutils.save_and_reraise_exception(): [ 2189.843770] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self.force_reraise() [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] raise self.value [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] updated_port = self._update_port( [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] _ensure_no_port_binding_failure(port) [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] raise exception.PortBindingFailed(port_id=port['id']) [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2189.844128] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] [ 2189.844128] env[62506]: INFO nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Terminating instance [ 2189.870779] env[62506]: DEBUG nova.network.neutron [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2189.984644] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.231s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2189.985268] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2189.988251] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.008s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2189.989604] env[62506]: INFO nova.compute.claims [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2189.992676] env[62506]: DEBUG nova.network.neutron [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2190.341772] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2190.348021] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2190.495414] env[62506]: DEBUG nova.compute.utils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2190.495414] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2190.495414] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2190.497829] env[62506]: DEBUG oslo_concurrency.lockutils [req-2f061188-7415-4d09-b969-cf0031baf175 req-20f5d7d3-87e1-4d37-8db0-2d4890d207d8 service nova] Releasing lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2190.500209] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquired lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2190.500520] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2190.616662] env[62506]: DEBUG nova.policy [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9389401046d64f4fa23f51a9fbefc899', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0fe07414118c4d82b70966bfb7b186ed', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2191.001950] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2191.027446] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2191.136523] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Successfully created port: feff2c99-b208-4429-9cc7-077296d38524 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2191.173268] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2191.186467] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd2a6abc-e608-4260-8ed9-53a3cd4fa79d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2191.196362] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b943586-be36-4694-912b-41e987612514 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2191.229913] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38a1af1a-c576-4498-8db5-73a5746924dc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2191.237680] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e150ddc5-4a8d-4b38-b798-d7f23e6d5caa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2191.251267] env[62506]: DEBUG nova.compute.provider_tree [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2191.371869] env[62506]: DEBUG nova.compute.manager [req-b1e0f4b1-06eb-4a8c-8b78-ef5679dceeca req-fb13bc97-c72c-4aec-8460-a91ed78196dd service nova] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Received event network-vif-deleted-d55540bb-f511-4fea-b6f2-93ba84855be0 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2191.676831] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Releasing lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2191.677291] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2191.677494] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2191.678339] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fb72c01c-95c5-4653-8da8-0195adcbfaff {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2191.687826] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d480aa65-d985-4c85-8ca2-739d3e1daca2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2191.712024] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5f77a974-8a53-439f-8700-736de9dba94e could not be found. [ 2191.712024] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2191.712024] env[62506]: INFO nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2191.712172] env[62506]: DEBUG oslo.service.loopingcall [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2191.712292] env[62506]: DEBUG nova.compute.manager [-] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2191.712384] env[62506]: DEBUG nova.network.neutron [-] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2191.731183] env[62506]: DEBUG nova.network.neutron [-] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2191.754278] env[62506]: DEBUG nova.scheduler.client.report [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2192.016472] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2192.040294] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2192.040587] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2192.040758] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2192.040932] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2192.041093] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2192.041350] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2192.041444] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2192.041598] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2192.041760] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2192.041918] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2192.042097] env[62506]: DEBUG nova.virt.hardware [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2192.042972] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db13f105-a5f8-458b-b64d-b4ad4c41703f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2192.050913] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae971991-07fe-4945-a3f9-6024652b2a85 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2192.064682] env[62506]: ERROR nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2192.064682] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2192.064682] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2192.064682] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2192.064682] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2192.064682] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2192.064682] env[62506]: ERROR nova.compute.manager raise self.value [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2192.064682] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2192.064682] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2192.064682] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2192.065205] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2192.065205] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2192.065205] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2192.065205] env[62506]: ERROR nova.compute.manager [ 2192.065205] env[62506]: Traceback (most recent call last): [ 2192.065205] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2192.065205] env[62506]: listener.cb(fileno) [ 2192.065205] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2192.065205] env[62506]: result = function(*args, **kwargs) [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2192.065205] env[62506]: return func(*args, **kwargs) [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2192.065205] env[62506]: raise e [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2192.065205] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2192.065205] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2192.065205] env[62506]: with excutils.save_and_reraise_exception(): [ 2192.065205] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2192.065205] env[62506]: self.force_reraise() [ 2192.065205] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2192.065205] env[62506]: raise self.value [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2192.065205] env[62506]: updated_port = self._update_port( [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2192.065205] env[62506]: _ensure_no_port_binding_failure(port) [ 2192.065205] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2192.065205] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2192.066225] env[62506]: nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2192.066225] env[62506]: Removing descriptor: 15 [ 2192.066225] env[62506]: ERROR nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Traceback (most recent call last): [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] yield resources [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self.driver.spawn(context, instance, image_meta, [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2192.066225] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] vm_ref = self.build_virtual_machine(instance, [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] vif_infos = vmwarevif.get_vif_info(self._session, [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] for vif in network_info: [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return self._sync_wrapper(fn, *args, **kwargs) [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self.wait() [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self[:] = self._gt.wait() [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return self._exit_event.wait() [ 2192.066596] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] result = hub.switch() [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return self.greenlet.switch() [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] result = function(*args, **kwargs) [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return func(*args, **kwargs) [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] raise e [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] nwinfo = self.network_api.allocate_for_instance( [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2192.066972] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] created_port_ids = self._update_ports_for_instance( [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] with excutils.save_and_reraise_exception(): [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self.force_reraise() [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] raise self.value [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] updated_port = self._update_port( [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] _ensure_no_port_binding_failure(port) [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2192.067370] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] raise exception.PortBindingFailed(port_id=port['id']) [ 2192.067733] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2192.067733] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] [ 2192.067733] env[62506]: INFO nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Terminating instance [ 2192.233416] env[62506]: DEBUG nova.network.neutron [-] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2192.258729] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.270s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2192.259258] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2192.261795] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.506s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2192.264035] env[62506]: INFO nova.compute.claims [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2192.569336] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2192.569651] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquired lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2192.569688] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2192.736128] env[62506]: INFO nova.compute.manager [-] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Took 1.02 seconds to deallocate network for instance. [ 2192.740528] env[62506]: DEBUG nova.compute.claims [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2192.740731] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2192.767454] env[62506]: DEBUG nova.compute.utils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2192.771076] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2192.771076] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2192.814391] env[62506]: DEBUG nova.policy [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3c9c372663b34fcaa0dac3e1c657e044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fc10dc6a19fb41cdbf046f6da6dc69fe', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2193.088492] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2193.125646] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Successfully created port: f58ea1b1-38ed-4cfc-8099-fbb2acad0812 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2193.210976] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2193.271342] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2193.397355] env[62506]: DEBUG nova.compute.manager [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Received event network-changed-feff2c99-b208-4429-9cc7-077296d38524 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2193.397614] env[62506]: DEBUG nova.compute.manager [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Refreshing instance network info cache due to event network-changed-feff2c99-b208-4429-9cc7-077296d38524. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2193.397738] env[62506]: DEBUG oslo_concurrency.lockutils [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] Acquiring lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2193.447965] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9aa83c4-a758-4675-982d-fa94d94e7649 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2193.457819] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e71cd081-6b48-46ed-b922-7a2e33e52143 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2193.490866] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6fbe90b-3356-4036-8cbb-c1be58a811e5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2193.498203] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85fe817a-c62d-44f5-a275-843da7441076 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2193.511384] env[62506]: DEBUG nova.compute.provider_tree [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2193.713736] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Releasing lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2193.714233] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2193.714576] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2193.714935] env[62506]: DEBUG oslo_concurrency.lockutils [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] Acquired lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2193.715137] env[62506]: DEBUG nova.network.neutron [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Refreshing network info cache for port feff2c99-b208-4429-9cc7-077296d38524 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2193.716558] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3759d0e5-bcd0-482b-81e5-c28447c6d07c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2193.725675] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-553ee90c-54ea-4caa-9cb2-369fe0238d06 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2193.748657] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 852705fb-d732-4c79-ba20-58f04806f15c could not be found. [ 2193.748894] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2193.749093] env[62506]: INFO nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2193.749345] env[62506]: DEBUG oslo.service.loopingcall [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2193.749570] env[62506]: DEBUG nova.compute.manager [-] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2193.749664] env[62506]: DEBUG nova.network.neutron [-] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2193.775347] env[62506]: DEBUG nova.network.neutron [-] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2194.014163] env[62506]: DEBUG nova.scheduler.client.report [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2194.093466] env[62506]: ERROR nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2194.093466] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2194.093466] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2194.093466] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2194.093466] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2194.093466] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2194.093466] env[62506]: ERROR nova.compute.manager raise self.value [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2194.093466] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2194.093466] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2194.093466] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2194.094269] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2194.094269] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2194.094269] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2194.094269] env[62506]: ERROR nova.compute.manager [ 2194.094269] env[62506]: Traceback (most recent call last): [ 2194.094269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2194.094269] env[62506]: listener.cb(fileno) [ 2194.094269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2194.094269] env[62506]: result = function(*args, **kwargs) [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2194.094269] env[62506]: return func(*args, **kwargs) [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2194.094269] env[62506]: raise e [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2194.094269] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2194.094269] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2194.094269] env[62506]: with excutils.save_and_reraise_exception(): [ 2194.094269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2194.094269] env[62506]: self.force_reraise() [ 2194.094269] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2194.094269] env[62506]: raise self.value [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2194.094269] env[62506]: updated_port = self._update_port( [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2194.094269] env[62506]: _ensure_no_port_binding_failure(port) [ 2194.094269] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2194.094269] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2194.095616] env[62506]: nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2194.095616] env[62506]: Removing descriptor: 15 [ 2194.235347] env[62506]: DEBUG nova.network.neutron [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2194.277389] env[62506]: DEBUG nova.network.neutron [-] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2194.283048] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2194.306892] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2194.307201] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2194.307394] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2194.307582] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2194.307726] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2194.307871] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2194.308098] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2194.308260] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2194.308427] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2194.308585] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2194.308750] env[62506]: DEBUG nova.virt.hardware [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2194.309826] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4682056b-3365-45b6-95df-17d510db723a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2194.318148] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-883bc218-20b6-443e-af79-e38c2bcbf475 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2194.322417] env[62506]: DEBUG nova.network.neutron [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2194.333878] env[62506]: ERROR nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Traceback (most recent call last): [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] yield resources [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self.driver.spawn(context, instance, image_meta, [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] vm_ref = self.build_virtual_machine(instance, [ 2194.333878] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] vif_infos = vmwarevif.get_vif_info(self._session, [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] for vif in network_info: [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] return self._sync_wrapper(fn, *args, **kwargs) [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self.wait() [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self[:] = self._gt.wait() [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] return self._exit_event.wait() [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2194.334362] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] current.throw(*self._exc) [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] result = function(*args, **kwargs) [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] return func(*args, **kwargs) [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] raise e [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] nwinfo = self.network_api.allocate_for_instance( [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] created_port_ids = self._update_ports_for_instance( [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] with excutils.save_and_reraise_exception(): [ 2194.334793] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self.force_reraise() [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] raise self.value [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] updated_port = self._update_port( [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] _ensure_no_port_binding_failure(port) [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] raise exception.PortBindingFailed(port_id=port['id']) [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2194.335204] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] [ 2194.335204] env[62506]: INFO nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Terminating instance [ 2194.519179] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.257s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2194.519725] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2194.522281] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 11.497s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2194.780035] env[62506]: INFO nova.compute.manager [-] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Took 1.03 seconds to deallocate network for instance. [ 2194.783530] env[62506]: DEBUG nova.compute.claims [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2194.783766] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2194.825053] env[62506]: DEBUG oslo_concurrency.lockutils [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] Releasing lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2194.825242] env[62506]: DEBUG nova.compute.manager [req-654abbe1-eb41-4c5a-9ad3-f2e3e56577bd req-3739ddb4-a8bc-4b78-8e97-65c6a1ac6535 service nova] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Received event network-vif-deleted-feff2c99-b208-4429-9cc7-077296d38524 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2194.837952] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2194.838132] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2194.838337] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2195.026427] env[62506]: DEBUG nova.compute.utils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2195.027734] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2195.028375] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2195.077105] env[62506]: DEBUG nova.policy [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e508d2263c0c47d5bccfb1e2f1409316', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7d771ac74ac54fb38a84337eed72348a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2195.356176] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Successfully created port: db2c4ae9-4eeb-40a1-8f60-58d230cec894 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2195.358029] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2195.430769] env[62506]: DEBUG nova.compute.manager [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Received event network-changed-f58ea1b1-38ed-4cfc-8099-fbb2acad0812 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2195.430928] env[62506]: DEBUG nova.compute.manager [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Refreshing instance network info cache due to event network-changed-f58ea1b1-38ed-4cfc-8099-fbb2acad0812. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2195.431311] env[62506]: DEBUG oslo_concurrency.lockutils [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] Acquiring lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2195.534098] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2195.543727] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2195.570591] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 5837199b-5937-4b7d-9c5b-e34c128db426 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2195.570674] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 45af80d9-96ec-41bd-8710-fc02c2963530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2195.570795] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 5f77a974-8a53-439f-8700-736de9dba94e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2195.570854] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 852705fb-d732-4c79-ba20-58f04806f15c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2195.571479] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 9e5fbbf6-2ace-4193-8add-5ae16c79e540 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2195.571479] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance dd2b0afa-7224-4d7e-8ff2-4142abcdef10 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2196.050395] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2196.050839] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2196.050993] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2196.051319] env[62506]: DEBUG oslo_concurrency.lockutils [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] Acquired lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2196.051496] env[62506]: DEBUG nova.network.neutron [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Refreshing network info cache for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2196.052429] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bd32ab21-e935-4804-ad3e-ad12d48595a7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2196.062911] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d6dcb09-96a9-4e55-b0ee-bca397761656 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2196.079752] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 26472ec8-3e42-42cf-b841-9d976f67afdd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2196.087345] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9e5fbbf6-2ace-4193-8add-5ae16c79e540 could not be found. [ 2196.087576] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2196.087755] env[62506]: INFO nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2196.087989] env[62506]: DEBUG oslo.service.loopingcall [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2196.088608] env[62506]: DEBUG nova.compute.manager [-] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2196.088608] env[62506]: DEBUG nova.network.neutron [-] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2196.107892] env[62506]: DEBUG nova.network.neutron [-] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2196.284729] env[62506]: ERROR nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2196.284729] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2196.284729] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2196.284729] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2196.284729] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2196.284729] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2196.284729] env[62506]: ERROR nova.compute.manager raise self.value [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2196.284729] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2196.284729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2196.284729] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2196.285276] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2196.285276] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2196.285276] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2196.285276] env[62506]: ERROR nova.compute.manager [ 2196.285276] env[62506]: Traceback (most recent call last): [ 2196.285276] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2196.285276] env[62506]: listener.cb(fileno) [ 2196.285276] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2196.285276] env[62506]: result = function(*args, **kwargs) [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2196.285276] env[62506]: return func(*args, **kwargs) [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2196.285276] env[62506]: raise e [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2196.285276] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2196.285276] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2196.285276] env[62506]: with excutils.save_and_reraise_exception(): [ 2196.285276] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2196.285276] env[62506]: self.force_reraise() [ 2196.285276] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2196.285276] env[62506]: raise self.value [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2196.285276] env[62506]: updated_port = self._update_port( [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2196.285276] env[62506]: _ensure_no_port_binding_failure(port) [ 2196.285276] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2196.285276] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2196.286207] env[62506]: nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2196.286207] env[62506]: Removing descriptor: 15 [ 2196.549752] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2196.570683] env[62506]: DEBUG nova.network.neutron [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2196.575432] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2196.575656] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2196.575813] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2196.576068] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2196.576273] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2196.576433] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2196.576645] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2196.576805] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2196.576973] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2196.577945] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2196.577945] env[62506]: DEBUG nova.virt.hardware [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2196.578221] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1155226d-3c57-4e5c-83f5-57c58365eaf6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2196.586407] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74c0128e-57ed-4cc8-9097-6502ddb85889 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2196.590636] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 84586d70-87bf-45a1-9f04-7ecfca539504 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2196.603066] env[62506]: ERROR nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Traceback (most recent call last): [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] yield resources [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self.driver.spawn(context, instance, image_meta, [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] vm_ref = self.build_virtual_machine(instance, [ 2196.603066] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] vif_infos = vmwarevif.get_vif_info(self._session, [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] for vif in network_info: [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] return self._sync_wrapper(fn, *args, **kwargs) [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self.wait() [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self[:] = self._gt.wait() [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] return self._exit_event.wait() [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2196.603583] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] current.throw(*self._exc) [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] result = function(*args, **kwargs) [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] return func(*args, **kwargs) [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] raise e [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] nwinfo = self.network_api.allocate_for_instance( [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] created_port_ids = self._update_ports_for_instance( [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] with excutils.save_and_reraise_exception(): [ 2196.604075] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self.force_reraise() [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] raise self.value [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] updated_port = self._update_port( [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] _ensure_no_port_binding_failure(port) [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] raise exception.PortBindingFailed(port_id=port['id']) [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2196.604514] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] [ 2196.604514] env[62506]: INFO nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Terminating instance [ 2196.610054] env[62506]: DEBUG nova.network.neutron [-] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2196.650154] env[62506]: DEBUG nova.network.neutron [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2197.093916] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e6a3f17c-e342-4445-8734-75e18b29b482 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2197.107422] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2197.107608] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2197.107792] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2197.112813] env[62506]: INFO nova.compute.manager [-] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Took 1.02 seconds to deallocate network for instance. [ 2197.114705] env[62506]: DEBUG nova.compute.claims [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2197.114892] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2197.153206] env[62506]: DEBUG oslo_concurrency.lockutils [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] Releasing lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2197.153502] env[62506]: DEBUG nova.compute.manager [req-fecafadd-6081-421a-a31d-5b4edd4b020c req-04b64f5d-8705-4213-a970-5154b88c8923 service nova] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Received event network-vif-deleted-f58ea1b1-38ed-4cfc-8099-fbb2acad0812 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2197.459062] env[62506]: DEBUG nova.compute.manager [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Received event network-changed-db2c4ae9-4eeb-40a1-8f60-58d230cec894 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2197.459281] env[62506]: DEBUG nova.compute.manager [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Refreshing instance network info cache due to event network-changed-db2c4ae9-4eeb-40a1-8f60-58d230cec894. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2197.459478] env[62506]: DEBUG oslo_concurrency.lockutils [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] Acquiring lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2197.596630] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 118f53cf-a170-44d3-bacd-9c94e98d8c5d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2197.596882] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2197.597047] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2197.624840] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2197.707812] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2197.715314] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-939c768c-7051-4339-b4a7-c193e5838ab6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2197.723746] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dacf4c70-dfc2-4424-b6c1-ac911670f16c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2197.753896] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47438f55-119d-4a9b-86e1-8d7f0925c91a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2197.760746] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb232715-a94b-4504-a778-b093afd093e5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2197.773688] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2198.210979] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2198.211462] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2198.211944] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2198.212060] env[62506]: DEBUG oslo_concurrency.lockutils [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] Acquired lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2198.212168] env[62506]: DEBUG nova.network.neutron [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Refreshing network info cache for port db2c4ae9-4eeb-40a1-8f60-58d230cec894 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2198.213778] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a3fe069d-7beb-4cfd-8704-efff35c3d794 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2198.222542] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21d54c4f-a968-4652-87ed-1197812e4b61 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2198.243598] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dd2b0afa-7224-4d7e-8ff2-4142abcdef10 could not be found. [ 2198.243809] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2198.243985] env[62506]: INFO nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2198.244236] env[62506]: DEBUG oslo.service.loopingcall [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2198.244483] env[62506]: DEBUG nova.compute.manager [-] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2198.244578] env[62506]: DEBUG nova.network.neutron [-] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2198.276706] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2198.395526] env[62506]: DEBUG nova.network.neutron [-] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2198.741025] env[62506]: DEBUG nova.network.neutron [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2198.781244] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2198.781504] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 4.259s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2198.781766] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.166s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2198.783214] env[62506]: INFO nova.compute.claims [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2198.814657] env[62506]: DEBUG nova.network.neutron [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2198.898080] env[62506]: DEBUG nova.network.neutron [-] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2199.317039] env[62506]: DEBUG oslo_concurrency.lockutils [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] Releasing lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2199.317500] env[62506]: DEBUG nova.compute.manager [req-302b83bc-298f-4ae7-8753-a2ea8660bcb0 req-58b36780-e310-4222-88cc-6e04df06fb3c service nova] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Received event network-vif-deleted-db2c4ae9-4eeb-40a1-8f60-58d230cec894 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2199.401099] env[62506]: INFO nova.compute.manager [-] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Took 1.16 seconds to deallocate network for instance. [ 2199.403728] env[62506]: DEBUG nova.compute.claims [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2199.404142] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2199.786190] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2199.786486] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2199.926601] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7434bc0-ee49-4776-ad09-3a2576294b2a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.935088] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-802b74cf-2018-4ce1-9364-4c7230597ecf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.965070] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b8ea77d-b440-42e4-9e01-b0dc15cf913e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.973093] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df0853f3-fc21-43c0-b9e9-8cc9c3ee378b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.986338] env[62506]: DEBUG nova.compute.provider_tree [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2200.293698] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2200.293868] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 2200.293986] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 2200.489788] env[62506]: DEBUG nova.scheduler.client.report [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2200.798709] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.798886] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.799026] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.799154] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.799275] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.799395] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.799525] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2200.799647] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Didn't find any instances for network info cache update. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10531}} [ 2200.799853] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2200.800014] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2200.800172] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2200.800330] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2200.800461] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 2200.995145] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.213s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2200.995636] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2200.998408] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.985s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2200.999795] env[62506]: INFO nova.compute.claims [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2201.505110] env[62506]: DEBUG nova.compute.utils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2201.507475] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2201.507642] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2201.546858] env[62506]: DEBUG nova.policy [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '083d0aee5b9343b2992cb2020e3acef1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '92286cfe20c34cb1b8f6bf106098d032', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2201.844946] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Successfully created port: 6a613470-16ae-461a-b2b6-37c241af4e05 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2202.008496] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2202.139013] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-470d73a6-58ce-4353-89f0-f20fdc554245 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2202.146663] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd8c18d7-4708-489b-becb-cf7ff1070d93 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2202.175804] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14b8c8e3-69d8-40da-8cf8-9c3a8b17ba61 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2202.182698] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dce1b9a-fc6b-4d63-b033-444909c2e8d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2202.195659] env[62506]: DEBUG nova.compute.provider_tree [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2202.540524] env[62506]: DEBUG nova.compute.manager [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Received event network-changed-6a613470-16ae-461a-b2b6-37c241af4e05 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2202.540856] env[62506]: DEBUG nova.compute.manager [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Refreshing instance network info cache due to event network-changed-6a613470-16ae-461a-b2b6-37c241af4e05. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2202.540930] env[62506]: DEBUG oslo_concurrency.lockutils [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] Acquiring lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2202.541093] env[62506]: DEBUG oslo_concurrency.lockutils [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] Acquired lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2202.541283] env[62506]: DEBUG nova.network.neutron [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Refreshing network info cache for port 6a613470-16ae-461a-b2b6-37c241af4e05 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2202.692364] env[62506]: ERROR nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2202.692364] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2202.692364] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2202.692364] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2202.692364] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2202.692364] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2202.692364] env[62506]: ERROR nova.compute.manager raise self.value [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2202.692364] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2202.692364] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2202.692364] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2202.692869] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2202.692869] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2202.692869] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2202.692869] env[62506]: ERROR nova.compute.manager [ 2202.692869] env[62506]: Traceback (most recent call last): [ 2202.692869] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2202.692869] env[62506]: listener.cb(fileno) [ 2202.692869] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2202.692869] env[62506]: result = function(*args, **kwargs) [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2202.692869] env[62506]: return func(*args, **kwargs) [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2202.692869] env[62506]: raise e [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2202.692869] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2202.692869] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2202.692869] env[62506]: with excutils.save_and_reraise_exception(): [ 2202.692869] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2202.692869] env[62506]: self.force_reraise() [ 2202.692869] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2202.692869] env[62506]: raise self.value [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2202.692869] env[62506]: updated_port = self._update_port( [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2202.692869] env[62506]: _ensure_no_port_binding_failure(port) [ 2202.692869] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2202.692869] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2202.693880] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2202.693880] env[62506]: Removing descriptor: 15 [ 2202.698590] env[62506]: DEBUG nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2203.022426] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2203.047416] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2203.047604] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2203.047761] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2203.047937] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2203.048090] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2203.048232] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2203.048431] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2203.048586] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2203.048745] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2203.048902] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2203.049080] env[62506]: DEBUG nova.virt.hardware [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2203.049935] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38b4fff8-b9e1-4e73-828f-3373c11e84bd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2203.057728] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4fa5a44-2dff-4039-bd9a-44bd5f7031ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2203.071542] env[62506]: ERROR nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Traceback (most recent call last): [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] yield resources [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self.driver.spawn(context, instance, image_meta, [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] vm_ref = self.build_virtual_machine(instance, [ 2203.071542] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] for vif in network_info: [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] return self._sync_wrapper(fn, *args, **kwargs) [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self.wait() [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self[:] = self._gt.wait() [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] return self._exit_event.wait() [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2203.071987] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] current.throw(*self._exc) [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] result = function(*args, **kwargs) [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] return func(*args, **kwargs) [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] raise e [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] nwinfo = self.network_api.allocate_for_instance( [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] created_port_ids = self._update_ports_for_instance( [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] with excutils.save_and_reraise_exception(): [ 2203.072431] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self.force_reraise() [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] raise self.value [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] updated_port = self._update_port( [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] _ensure_no_port_binding_failure(port) [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] raise exception.PortBindingFailed(port_id=port['id']) [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2203.072865] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] [ 2203.072865] env[62506]: INFO nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Terminating instance [ 2203.073681] env[62506]: DEBUG nova.network.neutron [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2203.154551] env[62506]: DEBUG nova.network.neutron [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2203.203259] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.205s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2203.203804] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2203.206264] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.161s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2203.207595] env[62506]: INFO nova.compute.claims [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2203.577200] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2203.657053] env[62506]: DEBUG oslo_concurrency.lockutils [req-38c2e497-b37c-45a7-b712-58d33c4d5d21 req-71faaee0-5f20-4634-b710-42f6a5e8558c service nova] Releasing lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2203.657465] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquired lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2203.657690] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2203.711663] env[62506]: DEBUG nova.compute.utils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2203.715053] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2203.715053] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2203.754387] env[62506]: DEBUG nova.policy [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cd88eb3cd85347b6a4ce836db22a46ae', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ea555e9a50274fbf9cc424f78240d4d6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2204.051557] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Successfully created port: eb8493e9-d211-43e8-a5b5-2d2cf0605f17 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2204.177467] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2204.220526] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2204.311795] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2204.425473] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-028d7a0e-78db-4593-a16d-6cd0f6ca3b6f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.433514] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27316354-6fe6-4ba1-a94d-b982302db631 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.466180] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c1eef77-1275-4d50-b2d4-8613190f0cda {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.473623] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e632a0-7328-4978-91f1-f70a452ffecf {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.487262] env[62506]: DEBUG nova.compute.provider_tree [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2204.568743] env[62506]: DEBUG nova.compute.manager [req-f1b05a87-97b4-4e70-ac1f-1cb555c7de72 req-48b0d14f-9f96-4620-be40-30b9d444aebe service nova] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Received event network-vif-deleted-6a613470-16ae-461a-b2b6-37c241af4e05 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2204.814565] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Releasing lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2204.814966] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2204.815174] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2204.815457] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-17cadc3f-6d63-4c42-b59f-c9e5289f1f0d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.824303] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59b2725b-a7ac-49c0-82c2-d858bcd4a316 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.846326] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 26472ec8-3e42-42cf-b841-9d976f67afdd could not be found. [ 2204.846563] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2204.846749] env[62506]: INFO nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2204.846987] env[62506]: DEBUG oslo.service.loopingcall [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2204.847226] env[62506]: DEBUG nova.compute.manager [-] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2204.847373] env[62506]: DEBUG nova.network.neutron [-] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2204.871009] env[62506]: DEBUG nova.network.neutron [-] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2204.990265] env[62506]: DEBUG nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2205.014281] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2205.014281] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2205.014281] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2205.014281] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2205.014281] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2205.014281] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2205.014281] env[62506]: ERROR nova.compute.manager raise self.value [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2205.014281] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2205.014281] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2205.014281] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2205.015078] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2205.015078] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2205.015078] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2205.015078] env[62506]: ERROR nova.compute.manager [ 2205.015078] env[62506]: Traceback (most recent call last): [ 2205.015078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2205.015078] env[62506]: listener.cb(fileno) [ 2205.015078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2205.015078] env[62506]: result = function(*args, **kwargs) [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2205.015078] env[62506]: return func(*args, **kwargs) [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2205.015078] env[62506]: raise e [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2205.015078] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2205.015078] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2205.015078] env[62506]: with excutils.save_and_reraise_exception(): [ 2205.015078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2205.015078] env[62506]: self.force_reraise() [ 2205.015078] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2205.015078] env[62506]: raise self.value [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2205.015078] env[62506]: updated_port = self._update_port( [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2205.015078] env[62506]: _ensure_no_port_binding_failure(port) [ 2205.015078] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2205.015078] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2205.016187] env[62506]: nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2205.016187] env[62506]: Removing descriptor: 15 [ 2205.230645] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2205.256056] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2205.256056] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2205.256252] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2205.256349] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2205.256527] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2205.256740] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2205.256961] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2205.257142] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2205.257312] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2205.257476] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2205.257653] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2205.258508] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1996c9c-191e-44b7-a3d1-7f2302c659ec {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.267685] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ff7af8f-0753-4333-9a60-554444edc753 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.282360] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Traceback (most recent call last): [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] yield resources [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self.driver.spawn(context, instance, image_meta, [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] vm_ref = self.build_virtual_machine(instance, [ 2205.282360] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] vif_infos = vmwarevif.get_vif_info(self._session, [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] for vif in network_info: [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] return self._sync_wrapper(fn, *args, **kwargs) [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self.wait() [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self[:] = self._gt.wait() [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] return self._exit_event.wait() [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2205.282857] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] current.throw(*self._exc) [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] result = function(*args, **kwargs) [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] return func(*args, **kwargs) [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] raise e [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] nwinfo = self.network_api.allocate_for_instance( [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] created_port_ids = self._update_ports_for_instance( [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] with excutils.save_and_reraise_exception(): [ 2205.283308] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self.force_reraise() [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] raise self.value [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] updated_port = self._update_port( [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] _ensure_no_port_binding_failure(port) [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] raise exception.PortBindingFailed(port_id=port['id']) [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2205.283779] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] [ 2205.283779] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Terminating instance [ 2205.372980] env[62506]: DEBUG nova.network.neutron [-] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2205.495787] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.289s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2205.496332] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2205.498936] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.272s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2205.786965] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2205.787175] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquired lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2205.787361] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2205.877811] env[62506]: INFO nova.compute.manager [-] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Took 1.03 seconds to deallocate network for instance. [ 2205.880018] env[62506]: DEBUG nova.compute.claims [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2205.880188] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2206.006941] env[62506]: DEBUG nova.compute.utils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2206.008877] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2206.008877] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2206.062501] env[62506]: DEBUG nova.policy [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cd88eb3cd85347b6a4ce836db22a46ae', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ea555e9a50274fbf9cc424f78240d4d6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2206.128903] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96d2eb84-076a-4dfa-8696-39e955165f80 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2206.136488] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f892dda9-60a7-45b5-ba96-5dbb1a36c8af {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2206.166529] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb70baae-31c6-4284-96e6-c9c298941ec1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2206.173884] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b656b9d9-b203-400c-b29d-9e973720ed63 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2206.186677] env[62506]: DEBUG nova.compute.provider_tree [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2206.306440] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2206.354064] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Successfully created port: b41b5aaf-d244-421d-9703-e82a1a0deed9 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2206.423484] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2206.511799] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2206.606986] env[62506]: DEBUG nova.compute.manager [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Received event network-changed-eb8493e9-d211-43e8-a5b5-2d2cf0605f17 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2206.607221] env[62506]: DEBUG nova.compute.manager [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Refreshing instance network info cache due to event network-changed-eb8493e9-d211-43e8-a5b5-2d2cf0605f17. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2206.607443] env[62506]: DEBUG oslo_concurrency.lockutils [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] Acquiring lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2206.689621] env[62506]: DEBUG nova.scheduler.client.report [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2206.926337] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Releasing lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2206.926774] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2206.926975] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2206.927304] env[62506]: DEBUG oslo_concurrency.lockutils [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] Acquired lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2206.927490] env[62506]: DEBUG nova.network.neutron [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Refreshing network info cache for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2206.928570] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ef3fbe7b-d4c0-445a-aee1-fd1188eba7e3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2206.938708] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7becc0e6-6485-405a-83ea-b687fe3a67f1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2206.960916] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 84586d70-87bf-45a1-9f04-7ecfca539504 could not be found. [ 2206.961153] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2206.961360] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2206.961612] env[62506]: DEBUG oslo.service.loopingcall [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2206.961831] env[62506]: DEBUG nova.compute.manager [-] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2206.961927] env[62506]: DEBUG nova.network.neutron [-] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2206.978720] env[62506]: DEBUG nova.network.neutron [-] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2207.196915] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.698s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2207.197589] env[62506]: ERROR nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Traceback (most recent call last): [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self.driver.spawn(context, instance, image_meta, [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] vm_ref = self.build_virtual_machine(instance, [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] vif_infos = vmwarevif.get_vif_info(self._session, [ 2207.197589] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] for vif in network_info: [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return self._sync_wrapper(fn, *args, **kwargs) [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self.wait() [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self[:] = self._gt.wait() [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return self._exit_event.wait() [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] result = hub.switch() [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2207.198014] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return self.greenlet.switch() [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] result = function(*args, **kwargs) [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] return func(*args, **kwargs) [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] raise e [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] nwinfo = self.network_api.allocate_for_instance( [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] created_port_ids = self._update_ports_for_instance( [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] with excutils.save_and_reraise_exception(): [ 2207.198665] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] self.force_reraise() [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] raise self.value [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] updated_port = self._update_port( [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] _ensure_no_port_binding_failure(port) [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] raise exception.PortBindingFailed(port_id=port['id']) [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] nova.exception.PortBindingFailed: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. [ 2207.199304] env[62506]: ERROR nova.compute.manager [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] [ 2207.199790] env[62506]: DEBUG nova.compute.utils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2207.199790] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.246s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2207.202668] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Build of instance 5837199b-5937-4b7d-9c5b-e34c128db426 was re-scheduled: Binding failed for port 48f1ba4c-87e1-4e7d-9eea-80ba2cff0ff3, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2207.203117] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2207.203317] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquiring lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2207.203494] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Acquired lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2207.204098] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2207.234729] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2207.234729] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2207.234729] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2207.234729] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2207.234729] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2207.234729] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2207.234729] env[62506]: ERROR nova.compute.manager raise self.value [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2207.234729] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2207.234729] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2207.234729] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2207.235333] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2207.235333] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2207.235333] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2207.235333] env[62506]: ERROR nova.compute.manager [ 2207.235333] env[62506]: Traceback (most recent call last): [ 2207.235333] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2207.235333] env[62506]: listener.cb(fileno) [ 2207.235333] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2207.235333] env[62506]: result = function(*args, **kwargs) [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2207.235333] env[62506]: return func(*args, **kwargs) [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2207.235333] env[62506]: raise e [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2207.235333] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2207.235333] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2207.235333] env[62506]: with excutils.save_and_reraise_exception(): [ 2207.235333] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2207.235333] env[62506]: self.force_reraise() [ 2207.235333] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2207.235333] env[62506]: raise self.value [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2207.235333] env[62506]: updated_port = self._update_port( [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2207.235333] env[62506]: _ensure_no_port_binding_failure(port) [ 2207.235333] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2207.235333] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2207.236332] env[62506]: nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2207.236332] env[62506]: Removing descriptor: 15 [ 2207.446527] env[62506]: DEBUG nova.network.neutron [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2207.480754] env[62506]: DEBUG nova.network.neutron [-] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2207.517071] env[62506]: DEBUG nova.network.neutron [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2207.520383] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2207.544270] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2207.544528] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2207.544703] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2207.544892] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2207.545052] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2207.545203] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2207.545410] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2207.545570] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2207.545737] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2207.545903] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2207.546091] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2207.547165] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a98b86a4-4ff0-4a07-8395-35432927a400 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2207.554781] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9b19bd9-f307-4917-86f3-39b45dd802a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2207.569947] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Traceback (most recent call last): [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] yield resources [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self.driver.spawn(context, instance, image_meta, [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] vm_ref = self.build_virtual_machine(instance, [ 2207.569947] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] vif_infos = vmwarevif.get_vif_info(self._session, [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] for vif in network_info: [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] return self._sync_wrapper(fn, *args, **kwargs) [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self.wait() [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self[:] = self._gt.wait() [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] return self._exit_event.wait() [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2207.570463] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] current.throw(*self._exc) [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] result = function(*args, **kwargs) [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] return func(*args, **kwargs) [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] raise e [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] nwinfo = self.network_api.allocate_for_instance( [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] created_port_ids = self._update_ports_for_instance( [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] with excutils.save_and_reraise_exception(): [ 2207.570960] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self.force_reraise() [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] raise self.value [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] updated_port = self._update_port( [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] _ensure_no_port_binding_failure(port) [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] raise exception.PortBindingFailed(port_id=port['id']) [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2207.571430] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] [ 2207.571430] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Terminating instance [ 2207.722736] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2207.804332] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2207.829773] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82d8b588-4112-4625-b296-8b39f8dc2e97 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2207.837760] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29002888-e010-4614-9571-a41671839eae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2207.868505] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52a29941-36d8-4240-9b5b-ca2bec77e612 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2207.875694] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bf4ff62-d151-4f5e-b6a1-5b84809a0f8b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2207.888972] env[62506]: DEBUG nova.compute.provider_tree [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2207.984500] env[62506]: INFO nova.compute.manager [-] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Took 1.02 seconds to deallocate network for instance. [ 2207.986936] env[62506]: DEBUG nova.compute.claims [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2207.987187] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2208.019810] env[62506]: DEBUG oslo_concurrency.lockutils [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] Releasing lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2208.020109] env[62506]: DEBUG nova.compute.manager [req-679e3fb6-11f3-4a96-906f-63ebaae0f572 req-0e57bad1-bc1a-45f8-af44-9f557861c02d service nova] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Received event network-vif-deleted-eb8493e9-d211-43e8-a5b5-2d2cf0605f17 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2208.074410] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2208.074650] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquired lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2208.074845] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2208.306804] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Releasing lock "refresh_cache-5837199b-5937-4b7d-9c5b-e34c128db426" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2208.307051] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2208.307243] env[62506]: DEBUG nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2208.307413] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2208.321313] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2208.392336] env[62506]: DEBUG nova.scheduler.client.report [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2208.592013] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2208.633526] env[62506]: DEBUG nova.compute.manager [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Received event network-changed-b41b5aaf-d244-421d-9703-e82a1a0deed9 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2208.633688] env[62506]: DEBUG nova.compute.manager [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Refreshing instance network info cache due to event network-changed-b41b5aaf-d244-421d-9703-e82a1a0deed9. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2208.633939] env[62506]: DEBUG oslo_concurrency.lockutils [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] Acquiring lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2208.671905] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2208.823391] env[62506]: DEBUG nova.network.neutron [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2208.897198] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.698s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2208.897884] env[62506]: ERROR nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Traceback (most recent call last): [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self.driver.spawn(context, instance, image_meta, [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] vm_ref = self.build_virtual_machine(instance, [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] vif_infos = vmwarevif.get_vif_info(self._session, [ 2208.897884] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] for vif in network_info: [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] return self._sync_wrapper(fn, *args, **kwargs) [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self.wait() [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self[:] = self._gt.wait() [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] return self._exit_event.wait() [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] current.throw(*self._exc) [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2208.898261] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] result = function(*args, **kwargs) [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] return func(*args, **kwargs) [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] raise e [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] nwinfo = self.network_api.allocate_for_instance( [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] created_port_ids = self._update_ports_for_instance( [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] with excutils.save_and_reraise_exception(): [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] self.force_reraise() [ 2208.898658] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] raise self.value [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] updated_port = self._update_port( [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] _ensure_no_port_binding_failure(port) [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] raise exception.PortBindingFailed(port_id=port['id']) [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] nova.exception.PortBindingFailed: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. [ 2208.899084] env[62506]: ERROR nova.compute.manager [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] [ 2208.899084] env[62506]: DEBUG nova.compute.utils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2208.900241] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.559s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2208.901683] env[62506]: INFO nova.compute.claims [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2208.905011] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Build of instance 45af80d9-96ec-41bd-8710-fc02c2963530 was re-scheduled: Binding failed for port b5e119fa-5e3c-46d5-8dd6-99d34b12c3fe, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2208.905466] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2208.905696] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquiring lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2208.905866] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Acquired lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2208.906091] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2209.174307] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Releasing lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2209.174858] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2209.175074] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2209.175389] env[62506]: DEBUG oslo_concurrency.lockutils [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] Acquired lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2209.175564] env[62506]: DEBUG nova.network.neutron [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Refreshing network info cache for port b41b5aaf-d244-421d-9703-e82a1a0deed9 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2209.176614] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3db85eb7-4e64-4227-9616-5fb58b7cab28 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2209.186536] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7372a381-8c19-4bda-bbc2-0d4e8f1bed39 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2209.207927] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e6a3f17c-e342-4445-8734-75e18b29b482 could not be found. [ 2209.208123] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2209.208304] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2209.208596] env[62506]: DEBUG oslo.service.loopingcall [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2209.208812] env[62506]: DEBUG nova.compute.manager [-] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2209.208906] env[62506]: DEBUG nova.network.neutron [-] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2209.223706] env[62506]: DEBUG nova.network.neutron [-] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2209.326417] env[62506]: INFO nova.compute.manager [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] [instance: 5837199b-5937-4b7d-9c5b-e34c128db426] Took 1.02 seconds to deallocate network for instance. [ 2209.423320] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2209.492435] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2209.695261] env[62506]: DEBUG nova.network.neutron [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2209.726171] env[62506]: DEBUG nova.network.neutron [-] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2209.788307] env[62506]: DEBUG nova.network.neutron [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2209.995158] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Releasing lock "refresh_cache-45af80d9-96ec-41bd-8710-fc02c2963530" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2209.995400] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2209.995588] env[62506]: DEBUG nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2209.995766] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2210.010614] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2210.052366] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbf96f95-f106-4f5c-95a0-1b96b86209d0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2210.061399] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5644ce1e-9382-4c3f-9122-382c671b005b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2210.090560] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73214a3c-d69e-471f-9f79-9ec3d70a3cb8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2210.097398] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98941daa-6d27-43bb-baa0-975760cf7484 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2210.110380] env[62506]: DEBUG nova.compute.provider_tree [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2210.228690] env[62506]: INFO nova.compute.manager [-] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Took 1.02 seconds to deallocate network for instance. [ 2210.231042] env[62506]: DEBUG nova.compute.claims [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2210.231318] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2210.290408] env[62506]: DEBUG oslo_concurrency.lockutils [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] Releasing lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2210.290752] env[62506]: DEBUG nova.compute.manager [req-e42657c6-26ad-466c-bc9e-cdb3262a5a42 req-e2f8400a-bdf4-4bb8-b2fe-177a4b2a154f service nova] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Received event network-vif-deleted-b41b5aaf-d244-421d-9703-e82a1a0deed9 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2210.357634] env[62506]: INFO nova.scheduler.client.report [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Deleted allocations for instance 5837199b-5937-4b7d-9c5b-e34c128db426 [ 2210.514857] env[62506]: DEBUG nova.network.neutron [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2210.612904] env[62506]: DEBUG nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2210.869184] env[62506]: DEBUG oslo_concurrency.lockutils [None req-32f0da2a-72ff-48f3-a0ad-8e647dc39fec tempest-ServersTestMultiNic-1534603799 tempest-ServersTestMultiNic-1534603799-project-member] Lock "5837199b-5937-4b7d-9c5b-e34c128db426" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.344s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2211.017607] env[62506]: INFO nova.compute.manager [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] [instance: 45af80d9-96ec-41bd-8710-fc02c2963530] Took 1.02 seconds to deallocate network for instance. [ 2211.118436] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.218s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2211.118978] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2211.121776] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.381s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2211.626873] env[62506]: DEBUG nova.compute.utils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2211.628579] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2211.628579] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2211.762197] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efbb4c3b-5539-425a-9bdd-b407ab056c2d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.770916] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b44270ac-1b91-45e8-91b5-8291a77f0ce0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.807760] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95afe49c-b407-457f-af0b-33ac106ffdee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.816337] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fa0c86c-5333-4270-a54e-ae0f496020ca {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2211.830856] env[62506]: DEBUG nova.compute.provider_tree [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2211.858950] env[62506]: DEBUG nova.policy [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cd88eb3cd85347b6a4ce836db22a46ae', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ea555e9a50274fbf9cc424f78240d4d6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2212.056958] env[62506]: INFO nova.scheduler.client.report [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Deleted allocations for instance 45af80d9-96ec-41bd-8710-fc02c2963530 [ 2212.134190] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2212.247275] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Successfully created port: 6e4b9262-ec95-43d2-a029-68e3b0cda03a {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2212.334143] env[62506]: DEBUG nova.scheduler.client.report [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2212.565608] env[62506]: DEBUG oslo_concurrency.lockutils [None req-94f2d0c9-a9ce-4119-ad20-8b550da06da0 tempest-ServerDiskConfigTestJSON-1472433509 tempest-ServerDiskConfigTestJSON-1472433509-project-member] Lock "45af80d9-96ec-41bd-8710-fc02c2963530" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 44.357s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2212.839484] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.718s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2212.840180] env[62506]: ERROR nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Traceback (most recent call last): [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self.driver.spawn(context, instance, image_meta, [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] vm_ref = self.build_virtual_machine(instance, [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] vif_infos = vmwarevif.get_vif_info(self._session, [ 2212.840180] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] for vif in network_info: [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] return self._sync_wrapper(fn, *args, **kwargs) [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self.wait() [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self[:] = self._gt.wait() [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] return self._exit_event.wait() [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] current.throw(*self._exc) [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2212.840625] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] result = function(*args, **kwargs) [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] return func(*args, **kwargs) [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] raise e [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] nwinfo = self.network_api.allocate_for_instance( [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] created_port_ids = self._update_ports_for_instance( [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] with excutils.save_and_reraise_exception(): [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] self.force_reraise() [ 2212.840981] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] raise self.value [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] updated_port = self._update_port( [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] _ensure_no_port_binding_failure(port) [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] raise exception.PortBindingFailed(port_id=port['id']) [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] nova.exception.PortBindingFailed: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. [ 2212.841542] env[62506]: ERROR nova.compute.manager [instance: 5f77a974-8a53-439f-8700-736de9dba94e] [ 2212.841542] env[62506]: DEBUG nova.compute.utils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2212.842269] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.058s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2212.849020] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Build of instance 5f77a974-8a53-439f-8700-736de9dba94e was re-scheduled: Binding failed for port d55540bb-f511-4fea-b6f2-93ba84855be0, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2212.849020] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2212.849020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2212.849020] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquired lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2212.849624] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2212.900680] env[62506]: DEBUG nova.compute.manager [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Received event network-changed-6e4b9262-ec95-43d2-a029-68e3b0cda03a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2212.900888] env[62506]: DEBUG nova.compute.manager [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Refreshing instance network info cache due to event network-changed-6e4b9262-ec95-43d2-a029-68e3b0cda03a. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2212.901116] env[62506]: DEBUG oslo_concurrency.lockutils [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] Acquiring lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2212.901263] env[62506]: DEBUG oslo_concurrency.lockutils [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] Acquired lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2212.901640] env[62506]: DEBUG nova.network.neutron [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Refreshing network info cache for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2213.140346] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2213.140346] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2213.140346] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2213.140346] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2213.140346] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2213.140346] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2213.140346] env[62506]: ERROR nova.compute.manager raise self.value [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2213.140346] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2213.140346] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2213.140346] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2213.140828] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2213.140828] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2213.140828] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2213.140828] env[62506]: ERROR nova.compute.manager [ 2213.140828] env[62506]: Traceback (most recent call last): [ 2213.140828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2213.140828] env[62506]: listener.cb(fileno) [ 2213.140828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2213.140828] env[62506]: result = function(*args, **kwargs) [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2213.140828] env[62506]: return func(*args, **kwargs) [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2213.140828] env[62506]: raise e [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2213.140828] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2213.140828] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2213.140828] env[62506]: with excutils.save_and_reraise_exception(): [ 2213.140828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2213.140828] env[62506]: self.force_reraise() [ 2213.140828] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2213.140828] env[62506]: raise self.value [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2213.140828] env[62506]: updated_port = self._update_port( [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2213.140828] env[62506]: _ensure_no_port_binding_failure(port) [ 2213.140828] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2213.140828] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2213.141671] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2213.141671] env[62506]: Removing descriptor: 15 [ 2213.145217] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2213.170750] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2213.171067] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2213.171237] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2213.171465] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2213.172420] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2213.172420] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2213.172420] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2213.172420] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2213.172420] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2213.172774] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2213.172774] env[62506]: DEBUG nova.virt.hardware [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2213.173970] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fe86844-32a8-4041-9058-74653d2995d7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.182036] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b826716f-347c-4b37-b52f-102b61c65135 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.197198] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Traceback (most recent call last): [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] yield resources [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self.driver.spawn(context, instance, image_meta, [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] vm_ref = self.build_virtual_machine(instance, [ 2213.197198] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] vif_infos = vmwarevif.get_vif_info(self._session, [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] for vif in network_info: [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] return self._sync_wrapper(fn, *args, **kwargs) [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self.wait() [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self[:] = self._gt.wait() [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] return self._exit_event.wait() [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2213.197662] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] current.throw(*self._exc) [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] result = function(*args, **kwargs) [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] return func(*args, **kwargs) [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] raise e [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] nwinfo = self.network_api.allocate_for_instance( [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] created_port_ids = self._update_ports_for_instance( [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] with excutils.save_and_reraise_exception(): [ 2213.198072] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self.force_reraise() [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] raise self.value [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] updated_port = self._update_port( [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] _ensure_no_port_binding_failure(port) [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] raise exception.PortBindingFailed(port_id=port['id']) [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2213.198451] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] [ 2213.198451] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Terminating instance [ 2213.424048] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2213.434476] env[62506]: DEBUG nova.network.neutron [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2213.483785] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a88ad5c-d971-4d77-9e65-ad465ab626a4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.491990] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfdaadfc-4408-4f02-9cff-93678c895f32 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.526999] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aea191d3-5d91-4f8a-b063-f8386a81b911 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.536349] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df7601ec-b3a3-4403-a598-6de4da32acec {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.554083] env[62506]: DEBUG nova.compute.provider_tree [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2213.651033] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2213.655763] env[62506]: DEBUG nova.network.neutron [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2213.701153] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2214.057553] env[62506]: DEBUG nova.scheduler.client.report [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2214.154213] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Releasing lock "refresh_cache-5f77a974-8a53-439f-8700-736de9dba94e" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2214.154965] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2214.154965] env[62506]: DEBUG nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2214.154965] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2214.161549] env[62506]: DEBUG oslo_concurrency.lockutils [req-66179bbe-fe38-43d7-bbc1-6c03a57db37f req-363876aa-44ba-4f62-a1a0-b5927472fd53 service nova] Releasing lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2214.161952] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquired lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2214.162153] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2214.171592] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2214.568418] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.726s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2214.569116] env[62506]: ERROR nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Traceback (most recent call last): [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self.driver.spawn(context, instance, image_meta, [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] vm_ref = self.build_virtual_machine(instance, [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] vif_infos = vmwarevif.get_vif_info(self._session, [ 2214.569116] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] for vif in network_info: [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return self._sync_wrapper(fn, *args, **kwargs) [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self.wait() [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self[:] = self._gt.wait() [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return self._exit_event.wait() [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] result = hub.switch() [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2214.569567] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return self.greenlet.switch() [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] result = function(*args, **kwargs) [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] return func(*args, **kwargs) [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] raise e [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] nwinfo = self.network_api.allocate_for_instance( [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] created_port_ids = self._update_ports_for_instance( [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] with excutils.save_and_reraise_exception(): [ 2214.570009] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] self.force_reraise() [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] raise self.value [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] updated_port = self._update_port( [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] _ensure_no_port_binding_failure(port) [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] raise exception.PortBindingFailed(port_id=port['id']) [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] nova.exception.PortBindingFailed: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. [ 2214.570440] env[62506]: ERROR nova.compute.manager [instance: 852705fb-d732-4c79-ba20-58f04806f15c] [ 2214.570793] env[62506]: DEBUG nova.compute.utils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2214.571856] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.457s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2214.579934] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Build of instance 852705fb-d732-4c79-ba20-58f04806f15c was re-scheduled: Binding failed for port feff2c99-b208-4429-9cc7-077296d38524, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2214.580314] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2214.580546] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquiring lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2214.580698] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Acquired lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2214.580852] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2214.674463] env[62506]: DEBUG nova.network.neutron [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2214.689739] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2214.855746] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2214.935509] env[62506]: DEBUG nova.compute.manager [req-3a9fe9e5-affe-456c-a754-4d50e8468436 req-ac5e8005-9e4d-44aa-af6c-3b250238dbd0 service nova] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Received event network-vif-deleted-6e4b9262-ec95-43d2-a029-68e3b0cda03a {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2215.119667] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2215.180416] env[62506]: INFO nova.compute.manager [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 5f77a974-8a53-439f-8700-736de9dba94e] Took 1.03 seconds to deallocate network for instance. [ 2215.218884] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51988c49-05b8-4d74-b433-9f5fe4bb1daa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2215.227693] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62d3042c-e68c-43e0-a9fc-58dace97e3bc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2215.260824] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b485a3b0-2cc8-422f-a949-a4e6fd4446a2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2215.268561] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bb1ecb9-7df2-483b-b468-452ca562b250 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2215.283070] env[62506]: DEBUG nova.compute.provider_tree [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2215.358127] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Releasing lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2215.358631] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2215.358754] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2215.359139] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-60e6caca-34b4-4d4c-83ba-daac891ff38d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2215.369556] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38a3535f-b905-45a8-9048-f7cf140b7396 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2215.398456] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 118f53cf-a170-44d3-bacd-9c94e98d8c5d could not be found. [ 2215.399797] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2215.400054] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2215.400349] env[62506]: DEBUG oslo.service.loopingcall [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2215.400610] env[62506]: DEBUG nova.compute.manager [-] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2215.401128] env[62506]: DEBUG nova.network.neutron [-] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2215.423123] env[62506]: DEBUG nova.network.neutron [-] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2215.438258] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2215.785948] env[62506]: DEBUG nova.scheduler.client.report [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2215.925690] env[62506]: DEBUG nova.network.neutron [-] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2215.943318] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Releasing lock "refresh_cache-852705fb-d732-4c79-ba20-58f04806f15c" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2215.943318] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2215.943318] env[62506]: DEBUG nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2215.943318] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2215.967782] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2216.214357] env[62506]: INFO nova.scheduler.client.report [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Deleted allocations for instance 5f77a974-8a53-439f-8700-736de9dba94e [ 2216.292577] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.721s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2216.293236] env[62506]: ERROR nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Traceback (most recent call last): [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self.driver.spawn(context, instance, image_meta, [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] vm_ref = self.build_virtual_machine(instance, [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] vif_infos = vmwarevif.get_vif_info(self._session, [ 2216.293236] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] for vif in network_info: [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] return self._sync_wrapper(fn, *args, **kwargs) [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self.wait() [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self[:] = self._gt.wait() [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] return self._exit_event.wait() [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] current.throw(*self._exc) [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2216.293675] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] result = function(*args, **kwargs) [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] return func(*args, **kwargs) [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] raise e [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] nwinfo = self.network_api.allocate_for_instance( [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] created_port_ids = self._update_ports_for_instance( [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] with excutils.save_and_reraise_exception(): [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] self.force_reraise() [ 2216.294124] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] raise self.value [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] updated_port = self._update_port( [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] _ensure_no_port_binding_failure(port) [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] raise exception.PortBindingFailed(port_id=port['id']) [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] nova.exception.PortBindingFailed: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. [ 2216.294556] env[62506]: ERROR nova.compute.manager [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] [ 2216.294556] env[62506]: DEBUG nova.compute.utils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2216.295424] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.891s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2216.298382] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Build of instance 9e5fbbf6-2ace-4193-8add-5ae16c79e540 was re-scheduled: Binding failed for port f58ea1b1-38ed-4cfc-8099-fbb2acad0812, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2216.298864] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2216.299059] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2216.299216] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2216.299378] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2216.429420] env[62506]: INFO nova.compute.manager [-] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Took 1.03 seconds to deallocate network for instance. [ 2216.434692] env[62506]: DEBUG nova.compute.claims [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2216.434963] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2216.471592] env[62506]: DEBUG nova.network.neutron [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2216.486621] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2216.486849] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2216.724891] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e8af13d3-e713-438f-ae4e-8305968c92b1 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "5f77a974-8a53-439f-8700-736de9dba94e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 44.533s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2216.828284] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2216.923163] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2393f7bd-d8bc-4a77-af2d-d2444ece3982 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2216.926951] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2216.933155] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4363aba-1e84-4f68-8d39-ee87941652e4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2216.963864] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f21a2b42-6395-4205-b6f5-52afee36ca99 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2216.972124] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1e43965-192e-4fbf-9b39-9ab392d6daca {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2216.976449] env[62506]: INFO nova.compute.manager [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] [instance: 852705fb-d732-4c79-ba20-58f04806f15c] Took 1.03 seconds to deallocate network for instance. [ 2216.990414] env[62506]: DEBUG nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2216.993707] env[62506]: DEBUG nova.compute.provider_tree [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2217.430478] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-9e5fbbf6-2ace-4193-8add-5ae16c79e540" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2217.430478] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2217.430478] env[62506]: DEBUG nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2217.430478] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2217.456313] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2217.497935] env[62506]: DEBUG nova.scheduler.client.report [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2217.519908] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2217.959771] env[62506]: DEBUG nova.network.neutron [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2218.004244] env[62506]: INFO nova.scheduler.client.report [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Deleted allocations for instance 852705fb-d732-4c79-ba20-58f04806f15c [ 2218.012373] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.716s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2218.012555] env[62506]: ERROR nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Traceback (most recent call last): [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self.driver.spawn(context, instance, image_meta, [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] vm_ref = self.build_virtual_machine(instance, [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] vif_infos = vmwarevif.get_vif_info(self._session, [ 2218.012555] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] for vif in network_info: [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] return self._sync_wrapper(fn, *args, **kwargs) [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self.wait() [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self[:] = self._gt.wait() [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] return self._exit_event.wait() [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] current.throw(*self._exc) [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2218.016293] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] result = function(*args, **kwargs) [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] return func(*args, **kwargs) [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] raise e [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] nwinfo = self.network_api.allocate_for_instance( [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] created_port_ids = self._update_ports_for_instance( [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] with excutils.save_and_reraise_exception(): [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] self.force_reraise() [ 2218.017208] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] raise self.value [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] updated_port = self._update_port( [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] _ensure_no_port_binding_failure(port) [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] raise exception.PortBindingFailed(port_id=port['id']) [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] nova.exception.PortBindingFailed: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. [ 2218.017574] env[62506]: ERROR nova.compute.manager [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] [ 2218.017574] env[62506]: DEBUG nova.compute.utils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2218.017856] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Build of instance dd2b0afa-7224-4d7e-8ff2-4142abcdef10 was re-scheduled: Binding failed for port db2c4ae9-4eeb-40a1-8f60-58d230cec894, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2218.017856] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2218.017856] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2218.017856] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2218.017856] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2218.020265] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.139s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2218.411328] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquiring lock "9bede4a7-88ad-4362-afd8-3cfbb71afecd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2218.411562] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Lock "9bede4a7-88ad-4362-afd8-3cfbb71afecd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2218.463230] env[62506]: INFO nova.compute.manager [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: 9e5fbbf6-2ace-4193-8add-5ae16c79e540] Took 1.03 seconds to deallocate network for instance. [ 2218.522242] env[62506]: DEBUG oslo_concurrency.lockutils [None req-676d4a50-3fc3-4667-a5b5-b96ea912310a tempest-AttachVolumeTestJSON-1510564766 tempest-AttachVolumeTestJSON-1510564766-project-member] Lock "852705fb-d732-4c79-ba20-58f04806f15c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.622s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2218.536692] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2218.637052] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2218.649243] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e25da8bc-ff8c-4e74-8ea2-f38347cab03c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2218.657631] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07a814f3-af26-4ac4-9aba-f980b4688260 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2218.696423] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6082ccef-8157-4825-b505-9c6ce6cde6b8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2218.704199] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58f48c8a-4f14-4b5b-b57b-73f47f514542 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2218.717621] env[62506]: DEBUG nova.compute.provider_tree [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2218.914116] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2219.139924] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-dd2b0afa-7224-4d7e-8ff2-4142abcdef10" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2219.139924] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2219.140259] env[62506]: DEBUG nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2219.140259] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2219.146343] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "38651166-97e3-467b-b4be-7002ecbd6396" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2219.146603] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "38651166-97e3-467b-b4be-7002ecbd6396" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2219.165113] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2219.220499] env[62506]: DEBUG nova.scheduler.client.report [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2219.436185] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2219.489778] env[62506]: INFO nova.scheduler.client.report [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Deleted allocations for instance 9e5fbbf6-2ace-4193-8add-5ae16c79e540 [ 2219.651800] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2219.668105] env[62506]: DEBUG nova.network.neutron [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2219.727326] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.706s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2219.727326] env[62506]: ERROR nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Traceback (most recent call last): [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self.driver.spawn(context, instance, image_meta, [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2219.727326] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] vm_ref = self.build_virtual_machine(instance, [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] for vif in network_info: [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] return self._sync_wrapper(fn, *args, **kwargs) [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self.wait() [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self[:] = self._gt.wait() [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] return self._exit_event.wait() [ 2219.727757] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] current.throw(*self._exc) [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] result = function(*args, **kwargs) [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] return func(*args, **kwargs) [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] raise e [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] nwinfo = self.network_api.allocate_for_instance( [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] created_port_ids = self._update_ports_for_instance( [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2219.728151] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] with excutils.save_and_reraise_exception(): [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] self.force_reraise() [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] raise self.value [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] updated_port = self._update_port( [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] _ensure_no_port_binding_failure(port) [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] raise exception.PortBindingFailed(port_id=port['id']) [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] nova.exception.PortBindingFailed: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. [ 2219.728511] env[62506]: ERROR nova.compute.manager [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] [ 2219.728879] env[62506]: DEBUG nova.compute.utils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2219.728879] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Build of instance 26472ec8-3e42-42cf-b841-9d976f67afdd was re-scheduled: Binding failed for port 6a613470-16ae-461a-b2b6-37c241af4e05, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2219.728879] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2219.728879] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquiring lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2219.729083] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Acquired lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2219.729083] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2219.729859] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.743s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2220.000708] env[62506]: DEBUG oslo_concurrency.lockutils [None req-0aafd348-76f0-4ee8-8905-91160e2cae9d tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "9e5fbbf6-2ace-4193-8add-5ae16c79e540" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 44.047s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2220.169756] env[62506]: INFO nova.compute.manager [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: dd2b0afa-7224-4d7e-8ff2-4142abcdef10] Took 1.03 seconds to deallocate network for instance. [ 2220.174193] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2220.254394] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2220.341809] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb447f1d-782e-45a7-9ec1-423d93d3ddef {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2220.350069] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae87e3bd-d97b-4c6d-b7b5-124776f6c716 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2220.384567] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ce7378e-c840-4d85-9be3-6050eb53b306 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2220.395183] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6917fda-65ad-4cfa-adf8-5b95039cf084 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2220.400172] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2220.412325] env[62506]: DEBUG nova.compute.provider_tree [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2220.904146] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Releasing lock "refresh_cache-26472ec8-3e42-42cf-b841-9d976f67afdd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2220.904496] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2220.904607] env[62506]: DEBUG nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2220.905384] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2220.914867] env[62506]: DEBUG nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2220.929020] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2221.199514] env[62506]: INFO nova.scheduler.client.report [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Deleted allocations for instance dd2b0afa-7224-4d7e-8ff2-4142abcdef10 [ 2221.419789] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.690s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2221.420452] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Traceback (most recent call last): [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self.driver.spawn(context, instance, image_meta, [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] vm_ref = self.build_virtual_machine(instance, [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] vif_infos = vmwarevif.get_vif_info(self._session, [ 2221.420452] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] for vif in network_info: [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] return self._sync_wrapper(fn, *args, **kwargs) [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self.wait() [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self[:] = self._gt.wait() [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] return self._exit_event.wait() [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] current.throw(*self._exc) [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2221.420829] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] result = function(*args, **kwargs) [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] return func(*args, **kwargs) [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] raise e [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] nwinfo = self.network_api.allocate_for_instance( [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] created_port_ids = self._update_ports_for_instance( [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] with excutils.save_and_reraise_exception(): [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] self.force_reraise() [ 2221.421225] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] raise self.value [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] updated_port = self._update_port( [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] _ensure_no_port_binding_failure(port) [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] raise exception.PortBindingFailed(port_id=port['id']) [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] nova.exception.PortBindingFailed: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. [ 2221.421598] env[62506]: ERROR nova.compute.manager [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] [ 2221.421598] env[62506]: DEBUG nova.compute.utils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2221.422564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.191s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2221.425374] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Build of instance 84586d70-87bf-45a1-9f04-7ecfca539504 was re-scheduled: Binding failed for port eb8493e9-d211-43e8-a5b5-2d2cf0605f17, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2221.425788] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2221.426024] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2221.426177] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquired lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2221.426333] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2221.431698] env[62506]: DEBUG nova.network.neutron [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2221.690938] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "df6be48d-bb76-4efd-9da5-3e6ce6467667" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2221.691265] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "df6be48d-bb76-4efd-9da5-3e6ce6467667" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2221.707214] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ab2b8a2a-cc54-4ce1-8691-01e950e07062 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "dd2b0afa-7224-4d7e-8ff2-4142abcdef10" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.978s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2221.935104] env[62506]: INFO nova.compute.manager [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] [instance: 26472ec8-3e42-42cf-b841-9d976f67afdd] Took 1.03 seconds to deallocate network for instance. [ 2221.949872] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2222.029714] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a5125fc-4478-4cc7-9a7c-2414ed4b63ba {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2222.033145] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2222.039516] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-106128ee-5d82-4f09-9292-fcd77a9f0c17 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2222.070385] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75380885-be5f-47c8-9277-f21fef10a87a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2222.077853] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e43883a9-be5a-4107-9931-a2c99f75456c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2222.092610] env[62506]: DEBUG nova.compute.provider_tree [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2222.193367] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2222.536049] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Releasing lock "refresh_cache-84586d70-87bf-45a1-9f04-7ecfca539504" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2222.536049] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2222.536049] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2222.536253] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2222.558623] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2222.595599] env[62506]: DEBUG nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2222.712872] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2222.958647] env[62506]: INFO nova.scheduler.client.report [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Deleted allocations for instance 26472ec8-3e42-42cf-b841-9d976f67afdd [ 2223.061267] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2223.099759] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.677s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2223.100386] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Traceback (most recent call last): [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self.driver.spawn(context, instance, image_meta, [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] vm_ref = self.build_virtual_machine(instance, [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] vif_infos = vmwarevif.get_vif_info(self._session, [ 2223.100386] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] for vif in network_info: [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] return self._sync_wrapper(fn, *args, **kwargs) [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self.wait() [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self[:] = self._gt.wait() [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] return self._exit_event.wait() [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] current.throw(*self._exc) [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2223.100801] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] result = function(*args, **kwargs) [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] return func(*args, **kwargs) [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] raise e [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] nwinfo = self.network_api.allocate_for_instance( [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] created_port_ids = self._update_ports_for_instance( [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] with excutils.save_and_reraise_exception(): [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] self.force_reraise() [ 2223.101242] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] raise self.value [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] updated_port = self._update_port( [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] _ensure_no_port_binding_failure(port) [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] raise exception.PortBindingFailed(port_id=port['id']) [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] nova.exception.PortBindingFailed: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. [ 2223.101665] env[62506]: ERROR nova.compute.manager [instance: e6a3f17c-e342-4445-8734-75e18b29b482] [ 2223.101665] env[62506]: DEBUG nova.compute.utils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2223.102150] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.667s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2223.105122] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Build of instance e6a3f17c-e342-4445-8734-75e18b29b482 was re-scheduled: Binding failed for port b41b5aaf-d244-421d-9703-e82a1a0deed9, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2223.105532] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2223.105757] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2223.105901] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquired lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2223.106070] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2223.420363] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2223.420610] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2223.466812] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fdc27bcd-1f1f-4fc0-b402-fab1aa122ee3 tempest-SecurityGroupsTestJSON-1947941450 tempest-SecurityGroupsTestJSON-1947941450-project-member] Lock "26472ec8-3e42-42cf-b841-9d976f67afdd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 37.870s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2223.564263] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 84586d70-87bf-45a1-9f04-7ecfca539504] Took 1.03 seconds to deallocate network for instance. [ 2223.610925] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2223.625284] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2223.701323] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13e679e8-9417-491d-986a-b7e6d11483fd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2223.707499] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2223.713191] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac3f0a9d-e14e-4e26-b1d1-0f18ccc6090c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2223.753506] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07db1f94-2fcd-4b0c-b852-31b44d286d49 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2223.761607] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8341f02c-aba7-4537-abf6-281dee92f91e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2223.774905] env[62506]: DEBUG nova.compute.provider_tree [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2223.923624] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2224.213235] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Releasing lock "refresh_cache-e6a3f17c-e342-4445-8734-75e18b29b482" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2224.213627] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2224.213913] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2224.214115] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2224.233637] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2224.278462] env[62506]: DEBUG nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2224.446604] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2224.591414] env[62506]: INFO nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Deleted allocations for instance 84586d70-87bf-45a1-9f04-7ecfca539504 [ 2224.735864] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2224.783293] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.681s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2224.783998] env[62506]: ERROR nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Traceback (most recent call last): [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self.driver.spawn(context, instance, image_meta, [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] vm_ref = self.build_virtual_machine(instance, [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] vif_infos = vmwarevif.get_vif_info(self._session, [ 2224.783998] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] for vif in network_info: [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] return self._sync_wrapper(fn, *args, **kwargs) [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self.wait() [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self[:] = self._gt.wait() [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] return self._exit_event.wait() [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] current.throw(*self._exc) [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2224.785039] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] result = function(*args, **kwargs) [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] return func(*args, **kwargs) [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] raise e [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] nwinfo = self.network_api.allocate_for_instance( [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] created_port_ids = self._update_ports_for_instance( [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] with excutils.save_and_reraise_exception(): [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] self.force_reraise() [ 2224.786417] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] raise self.value [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] updated_port = self._update_port( [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] _ensure_no_port_binding_failure(port) [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] raise exception.PortBindingFailed(port_id=port['id']) [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] nova.exception.PortBindingFailed: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. [ 2224.786885] env[62506]: ERROR nova.compute.manager [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] [ 2224.786885] env[62506]: DEBUG nova.compute.utils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2224.787270] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.266s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2224.788317] env[62506]: INFO nova.compute.claims [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2224.790271] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Build of instance 118f53cf-a170-44d3-bacd-9c94e98d8c5d was re-scheduled: Binding failed for port 6e4b9262-ec95-43d2-a029-68e3b0cda03a, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2224.790703] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2224.791225] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquiring lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2224.791225] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Acquired lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2224.791850] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2225.100721] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "84586d70-87bf-45a1-9f04-7ecfca539504" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 39.107s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2225.242065] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: e6a3f17c-e342-4445-8734-75e18b29b482] Took 1.02 seconds to deallocate network for instance. [ 2225.315055] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2225.402714] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2225.905433] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Releasing lock "refresh_cache-118f53cf-a170-44d3-bacd-9c94e98d8c5d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2225.905675] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2225.905863] env[62506]: DEBUG nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2225.906047] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2225.912788] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b60f4bc-8e01-4273-a108-0647f1339e12 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2225.920375] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3efa310-977a-44c5-8c16-a88d636536da {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2225.949823] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2225.951548] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dd745fa-c0b1-4530-b909-44e6cd06ddb5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2225.959291] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abd5b565-cc5e-49f6-99a1-04adabb8d983 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2225.974023] env[62506]: DEBUG nova.compute.provider_tree [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2226.265218] env[62506]: INFO nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Deleted allocations for instance e6a3f17c-e342-4445-8734-75e18b29b482 [ 2226.455504] env[62506]: DEBUG nova.network.neutron [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2226.477089] env[62506]: DEBUG nova.scheduler.client.report [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2226.776021] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "e6a3f17c-e342-4445-8734-75e18b29b482" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 40.750s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2226.961247] env[62506]: INFO nova.compute.manager [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] [instance: 118f53cf-a170-44d3-bacd-9c94e98d8c5d] Took 1.05 seconds to deallocate network for instance. [ 2226.983348] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.196s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2226.983348] env[62506]: DEBUG nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2226.985596] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.549s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2226.986892] env[62506]: INFO nova.compute.claims [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2227.491725] env[62506]: DEBUG nova.compute.utils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2227.493609] env[62506]: DEBUG nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Not allocating networking since 'none' was specified. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1984}} [ 2227.992850] env[62506]: INFO nova.scheduler.client.report [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Deleted allocations for instance 118f53cf-a170-44d3-bacd-9c94e98d8c5d [ 2227.998730] env[62506]: DEBUG nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2228.096238] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61506ecc-35c3-420a-b2ac-d3a4d000eed2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2228.104302] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2c0b0ec-ad71-488d-97fb-a2183a2fd60c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2228.140848] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2d8b4c8-b13a-4c25-b070-8dc38cc838f7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2228.148389] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06188749-f7bd-4926-b26f-5b970f2299fa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2228.165490] env[62506]: DEBUG nova.compute.provider_tree [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2228.508512] env[62506]: DEBUG oslo_concurrency.lockutils [None req-53d34904-fa96-48d6-b99b-ff9ec981b178 tempest-ListServersNegativeTestJSON-1225285423 tempest-ListServersNegativeTestJSON-1225285423-project-member] Lock "118f53cf-a170-44d3-bacd-9c94e98d8c5d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 42.449s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2228.668565] env[62506]: DEBUG nova.scheduler.client.report [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2229.010446] env[62506]: DEBUG nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2229.039799] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2229.039924] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2229.040097] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2229.040287] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2229.040436] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2229.040581] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2229.040861] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2229.040949] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2229.041128] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2229.041290] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2229.041468] env[62506]: DEBUG nova.virt.hardware [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2229.042389] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-069512c1-1738-4f3a-bdc9-ed9a2fb115e7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2229.050603] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-494fba05-44b8-4b42-b6b9-83f9fdda973d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2229.063849] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2229.069545] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Creating folder: Project (738506078f074e37959e3576afaf729e). Parent ref: group-v446953. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 2229.069846] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a4efad19-215a-4c47-ad4e-a27e75530eae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2229.079711] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Created folder: Project (738506078f074e37959e3576afaf729e) in parent group-v446953. [ 2229.079957] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Creating folder: Instances. Parent ref: group-v446984. {{(pid=62506) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 2229.080160] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4e970787-7eb4-4091-829d-dbc0a8d86b41 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2229.088969] env[62506]: INFO nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Created folder: Instances in parent group-v446984. [ 2229.089189] env[62506]: DEBUG oslo.service.loopingcall [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2229.089384] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2229.089583] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f205da3d-c3e2-47ef-a808-406c0a66d90c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2229.105574] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2229.105574] env[62506]: value = "task-2190898" [ 2229.105574] env[62506]: _type = "Task" [ 2229.105574] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2229.112939] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190898, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2229.176025] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.190s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2229.176025] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2229.180434] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.006s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2229.181900] env[62506]: INFO nova.compute.claims [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2229.615604] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190898, 'name': CreateVM_Task, 'duration_secs': 0.258612} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2229.615879] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2229.616490] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2229.616681] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2229.617009] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2229.617269] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-09287224-a6da-4bd9-98ad-7428f36aedbb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2229.622060] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2229.622060] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52d5bd86-10b0-097e-db98-0fb11f5b4252" [ 2229.622060] env[62506]: _type = "Task" [ 2229.622060] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2229.632862] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52d5bd86-10b0-097e-db98-0fb11f5b4252, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2229.687862] env[62506]: DEBUG nova.compute.utils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2229.691483] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2229.692054] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2229.732840] env[62506]: DEBUG nova.policy [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd56d7d7aeb864eb9927d4faecfc29139', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'af1b77d5a0ad428daf5889615381f779', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2230.132613] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52d5bd86-10b0-097e-db98-0fb11f5b4252, 'name': SearchDatastore_Task, 'duration_secs': 0.009173} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2230.133129] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2230.133205] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2230.133390] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2230.133606] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2230.133742] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2230.134076] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-074f5ae3-cd23-469e-be90-6f79f0b97c4f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.142235] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2230.142418] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2230.143129] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-279d20f5-5a50-4100-9930-57d5c6928c04 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.148830] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2230.148830] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]521eb7d9-ec02-a18f-4892-e59dd9d9e223" [ 2230.148830] env[62506]: _type = "Task" [ 2230.148830] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2230.159610] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]521eb7d9-ec02-a18f-4892-e59dd9d9e223, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2230.192503] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2230.207395] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Successfully created port: 59852802-810f-4bab-8fd4-f3cda8878d02 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2230.323765] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7a7b03a-40ea-435c-b192-97e141618ec3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.331930] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-833900a1-53ae-4c68-bd0a-53543de7f77c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.364274] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dead19df-c605-4ec8-b013-0997e10d4d94 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.372227] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0a0d967-bb68-4ada-9f27-33fe4725b33b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.386033] env[62506]: DEBUG nova.compute.provider_tree [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2230.391229] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquiring lock "54ea1f68-8c18-4564-9743-6aec7d1995e9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2230.391504] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Lock "54ea1f68-8c18-4564-9743-6aec7d1995e9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2230.660745] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]521eb7d9-ec02-a18f-4892-e59dd9d9e223, 'name': SearchDatastore_Task, 'duration_secs': 0.008938} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2230.661585] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-6555cae9-9c27-48eb-8986-afadc96e3f84 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2230.666739] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2230.666739] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]5272e4f2-640b-e28e-df7f-bff4899e7fc4" [ 2230.666739] env[62506]: _type = "Task" [ 2230.666739] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2230.674965] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5272e4f2-640b-e28e-df7f-bff4899e7fc4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2230.893211] env[62506]: DEBUG nova.scheduler.client.report [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2230.899657] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2231.037707] env[62506]: DEBUG nova.compute.manager [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Received event network-changed-59852802-810f-4bab-8fd4-f3cda8878d02 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2231.037707] env[62506]: DEBUG nova.compute.manager [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Refreshing instance network info cache due to event network-changed-59852802-810f-4bab-8fd4-f3cda8878d02. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2231.038126] env[62506]: DEBUG oslo_concurrency.lockutils [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] Acquiring lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2231.038126] env[62506]: DEBUG oslo_concurrency.lockutils [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] Acquired lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2231.038273] env[62506]: DEBUG nova.network.neutron [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Refreshing network info cache for port 59852802-810f-4bab-8fd4-f3cda8878d02 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2231.181824] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]5272e4f2-640b-e28e-df7f-bff4899e7fc4, 'name': SearchDatastore_Task, 'duration_secs': 0.008925} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2231.182138] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2231.182407] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2231.182668] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-aa584870-3a5e-407f-8cda-b12a5ca7b292 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2231.191360] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2231.191360] env[62506]: value = "task-2190899" [ 2231.191360] env[62506]: _type = "Task" [ 2231.191360] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2231.195295] env[62506]: ERROR nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2231.195295] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2231.195295] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2231.195295] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2231.195295] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2231.195295] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2231.195295] env[62506]: ERROR nova.compute.manager raise self.value [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2231.195295] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2231.195295] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2231.195295] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2231.195915] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2231.195915] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2231.195915] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2231.195915] env[62506]: ERROR nova.compute.manager [ 2231.195915] env[62506]: Traceback (most recent call last): [ 2231.195915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2231.195915] env[62506]: listener.cb(fileno) [ 2231.195915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2231.195915] env[62506]: result = function(*args, **kwargs) [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2231.195915] env[62506]: return func(*args, **kwargs) [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2231.195915] env[62506]: raise e [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2231.195915] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2231.195915] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2231.195915] env[62506]: with excutils.save_and_reraise_exception(): [ 2231.195915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2231.195915] env[62506]: self.force_reraise() [ 2231.195915] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2231.195915] env[62506]: raise self.value [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2231.195915] env[62506]: updated_port = self._update_port( [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2231.195915] env[62506]: _ensure_no_port_binding_failure(port) [ 2231.195915] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2231.195915] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2231.196912] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2231.196912] env[62506]: Removing descriptor: 19 [ 2231.200209] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190899, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2231.210863] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2231.236241] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2231.236494] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2231.236678] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2231.236873] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2231.237021] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2231.237175] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2231.237380] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2231.237538] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2231.237703] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2231.237863] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2231.238052] env[62506]: DEBUG nova.virt.hardware [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2231.238869] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea86edd1-2d2a-45c4-96c7-e96063a1bf4c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2231.247688] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f5c58eb-141d-4aa1-b42c-9380373e7ffb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2231.261376] env[62506]: ERROR nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Traceback (most recent call last): [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] yield resources [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self.driver.spawn(context, instance, image_meta, [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] vm_ref = self.build_virtual_machine(instance, [ 2231.261376] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] for vif in network_info: [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] return self._sync_wrapper(fn, *args, **kwargs) [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self.wait() [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self[:] = self._gt.wait() [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] return self._exit_event.wait() [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2231.261921] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] current.throw(*self._exc) [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] result = function(*args, **kwargs) [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] return func(*args, **kwargs) [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] raise e [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] nwinfo = self.network_api.allocate_for_instance( [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] created_port_ids = self._update_ports_for_instance( [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] with excutils.save_and_reraise_exception(): [ 2231.262412] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self.force_reraise() [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] raise self.value [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] updated_port = self._update_port( [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] _ensure_no_port_binding_failure(port) [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] raise exception.PortBindingFailed(port_id=port['id']) [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2231.262857] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] [ 2231.262857] env[62506]: INFO nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Terminating instance [ 2231.405502] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.225s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2231.405933] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2231.411650] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.699s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2231.413419] env[62506]: INFO nova.compute.claims [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2231.434510] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2231.557027] env[62506]: DEBUG nova.network.neutron [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2231.610497] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2231.640883] env[62506]: DEBUG nova.network.neutron [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2231.701982] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190899, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.483244} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2231.701982] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2231.701982] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2231.701982] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-2706100a-792d-43ce-8226-bb7b1e222e40 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2231.707417] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2231.707417] env[62506]: value = "task-2190900" [ 2231.707417] env[62506]: _type = "Task" [ 2231.707417] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2231.715242] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190900, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2231.768561] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquiring lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2231.914541] env[62506]: DEBUG nova.compute.utils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2231.914845] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2231.915023] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2231.991572] env[62506]: DEBUG nova.policy [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bbccae76c293437b84873cc656a055c3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a983195f14a44160a2cf5c170bd79ffd', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2232.114067] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2232.143953] env[62506]: DEBUG oslo_concurrency.lockutils [req-b12c1070-e201-4922-a345-74a4022aa82f req-1392435c-1955-47aa-a1df-a2d4f66a7e21 service nova] Releasing lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2232.144402] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquired lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2232.144617] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2232.221057] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190900, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.073438} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2232.221320] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2232.222214] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42c758d5-9308-4123-86c9-8c119c82ac28 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.246345] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Reconfiguring VM instance instance-0000005c to attach disk [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2232.246814] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-a1195122-37b1-4d67-8b80-91c889fdbf7f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.270974] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2232.270974] env[62506]: value = "task-2190901" [ 2232.270974] env[62506]: _type = "Task" [ 2232.270974] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2232.271739] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Successfully created port: fa808efa-bc76-4bfb-a71f-aa17ce0b4862 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2232.282533] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190901, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2232.417758] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2232.555325] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-521ffce8-005c-4162-a03d-4bb8234b60c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.564342] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cd80f49-70dd-460e-966f-8b7041bffa0e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.598986] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41df8d96-00b2-44e1-be77-07f0e92b27e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.606888] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad88aed7-9814-491e-bf40-ac7b3e7ea574 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.620223] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2232.664270] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2232.772755] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2232.784285] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190901, 'name': ReconfigVM_Task, 'duration_secs': 0.275563} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2232.784679] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Reconfigured VM instance instance-0000005c to attach disk [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2232.785333] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-47d8298d-113e-496c-be16-df9d0c056a04 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2232.792939] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2232.792939] env[62506]: value = "task-2190902" [ 2232.792939] env[62506]: _type = "Task" [ 2232.792939] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2232.801881] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190902, 'name': Rename_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2233.137675] env[62506]: DEBUG nova.compute.manager [req-9fedb7b8-0bd0-41b9-b94f-8cc91b4c7991 req-a5246378-94f5-4854-bfbf-23d479e07610 service nova] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Received event network-vif-deleted-59852802-810f-4bab-8fd4-f3cda8878d02 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2233.147251] env[62506]: ERROR nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [req-a0fe7378-987e-4b63-9c8d-6d1ccbb4185c] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-a0fe7378-987e-4b63-9c8d-6d1ccbb4185c"}]} [ 2233.166306] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 2233.190973] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 2233.190973] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2233.201540] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 2233.224352] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 2233.278972] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Releasing lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2233.279443] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2233.279645] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2233.280043] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-94a7e280-3434-411e-86f2-5dadafed4592 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.291510] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a27321f9-2298-49df-98c9-c4baa015405d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.320934] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9bede4a7-88ad-4362-afd8-3cfbb71afecd could not be found. [ 2233.321223] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2233.321445] env[62506]: INFO nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2233.321740] env[62506]: DEBUG oslo.service.loopingcall [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2233.327478] env[62506]: DEBUG nova.compute.manager [-] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2233.327703] env[62506]: DEBUG nova.network.neutron [-] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2233.329429] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190902, 'name': Rename_Task, 'duration_secs': 0.132395} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2233.329887] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2233.330440] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-27e02c03-b808-41fa-b5b4-a46e77a07a8c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.336651] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2233.336651] env[62506]: value = "task-2190903" [ 2233.336651] env[62506]: _type = "Task" [ 2233.336651] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2233.346162] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190903, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2233.346854] env[62506]: DEBUG nova.network.neutron [-] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2233.364489] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60f7e56c-d36f-4a7d-9157-ff0776bc027b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.372886] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a8036db-cbc9-48c5-8bc5-43e89fbe7a10 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.414324] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bb41fbb-29a5-49dc-b061-64b0e99deeab {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.417059] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-958409dd-1044-4a53-9546-9918dabafd65 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.422460] env[62506]: ERROR nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2233.422460] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2233.422460] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2233.422460] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2233.422460] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2233.422460] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2233.422460] env[62506]: ERROR nova.compute.manager raise self.value [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2233.422460] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2233.422460] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2233.422460] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2233.423014] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2233.423014] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2233.423014] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2233.423014] env[62506]: ERROR nova.compute.manager [ 2233.423014] env[62506]: Traceback (most recent call last): [ 2233.423014] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2233.423014] env[62506]: listener.cb(fileno) [ 2233.423014] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2233.423014] env[62506]: result = function(*args, **kwargs) [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2233.423014] env[62506]: return func(*args, **kwargs) [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2233.423014] env[62506]: raise e [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2233.423014] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2233.423014] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2233.423014] env[62506]: with excutils.save_and_reraise_exception(): [ 2233.423014] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2233.423014] env[62506]: self.force_reraise() [ 2233.423014] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2233.423014] env[62506]: raise self.value [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2233.423014] env[62506]: updated_port = self._update_port( [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2233.423014] env[62506]: _ensure_no_port_binding_failure(port) [ 2233.423014] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2233.423014] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2233.423967] env[62506]: nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2233.423967] env[62506]: Removing descriptor: 19 [ 2233.433031] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2233.435204] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2233.460041] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2233.460367] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2233.460585] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2233.460859] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2233.461295] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2233.461881] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2233.461881] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2233.461881] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2233.462069] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2233.462263] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2233.462441] env[62506]: DEBUG nova.virt.hardware [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2233.463535] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf7ba021-647c-4bff-b935-0cbe651f964b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.472208] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32b98b1c-008e-4267-8fb2-03e90467706f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.488238] env[62506]: ERROR nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Traceback (most recent call last): [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] yield resources [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self.driver.spawn(context, instance, image_meta, [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] vm_ref = self.build_virtual_machine(instance, [ 2233.488238] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] vif_infos = vmwarevif.get_vif_info(self._session, [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] for vif in network_info: [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] return self._sync_wrapper(fn, *args, **kwargs) [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self.wait() [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self[:] = self._gt.wait() [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] return self._exit_event.wait() [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2233.488721] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] current.throw(*self._exc) [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] result = function(*args, **kwargs) [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] return func(*args, **kwargs) [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] raise e [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] nwinfo = self.network_api.allocate_for_instance( [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] created_port_ids = self._update_ports_for_instance( [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] with excutils.save_and_reraise_exception(): [ 2233.489183] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self.force_reraise() [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] raise self.value [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] updated_port = self._update_port( [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] _ensure_no_port_binding_failure(port) [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] raise exception.PortBindingFailed(port_id=port['id']) [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2233.489578] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] [ 2233.489578] env[62506]: INFO nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Terminating instance [ 2233.848059] env[62506]: DEBUG oslo_vmware.api [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190903, 'name': PowerOnVM_Task, 'duration_secs': 0.405002} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2233.848331] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2233.848440] env[62506]: INFO nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Took 4.84 seconds to spawn the instance on the hypervisor. [ 2233.848622] env[62506]: DEBUG nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2233.849468] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85b8708d-f232-46b8-9035-4e11084b3c34 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2233.852174] env[62506]: DEBUG nova.network.neutron [-] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2233.946063] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquiring lock "2153994f-947b-47a5-8eca-f665c28eb87f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2233.946063] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Lock "2153994f-947b-47a5-8eca-f665c28eb87f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2233.958239] env[62506]: ERROR nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [req-8b2c09b4-c1f9-45c9-aa34-f8adf70cdf1f] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 432b599d-3cbd-4439-9dff-9721e0cca9b4. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-8b2c09b4-c1f9-45c9-aa34-f8adf70cdf1f"}]} [ 2233.975027] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Refreshing inventories for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:822}} [ 2233.992490] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2233.992666] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquired lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2233.993335] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2233.996390] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating ProviderTree inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:786}} [ 2233.997529] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2234.008230] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Refreshing aggregate associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, aggregates: None {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:831}} [ 2234.032579] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Refreshing trait associations for resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE,HW_ARCH_X86_64,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_SAME_HOST_COLD_MIGRATE {{(pid=62506) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:843}} [ 2234.142160] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d91f03a-ac53-442e-b71d-b9ddb8e86b03 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2234.150218] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75c6f320-d4c5-44b2-a057-fe44d05998e4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2234.181633] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3607f2c8-fdc7-4f20-86ad-babcfc182360 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2234.189598] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6de5cab3-73fb-404a-a113-fe41028b13ee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2234.204844] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2234.355105] env[62506]: INFO nova.compute.manager [-] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Took 1.03 seconds to deallocate network for instance. [ 2234.357267] env[62506]: DEBUG nova.compute.claims [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2234.357443] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2234.368896] env[62506]: INFO nova.compute.manager [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Took 16.87 seconds to build instance. [ 2234.447811] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2234.511019] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2234.582010] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2234.736386] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 116 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 2234.736673] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 116 to 117 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 2234.736926] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2234.821070] env[62506]: INFO nova.compute.manager [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Rebuilding instance [ 2234.861181] env[62506]: DEBUG nova.compute.manager [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2234.862038] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94613f54-5432-4dcc-9000-1c3c3d76fe17 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2234.868431] env[62506]: DEBUG oslo_concurrency.lockutils [None req-e0c5a7d6-0fb3-4a3b-9501-0641526b477e tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 18.382s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2234.967046] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2235.084628] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Releasing lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2235.084888] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2235.085109] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2235.085416] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5668cb8c-fa22-4282-97a5-7a5ea64ee368 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2235.094389] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b8f2ad7-c7e8-4ade-af8f-d2d4f2922a3c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2235.116814] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 38651166-97e3-467b-b4be-7002ecbd6396 could not be found. [ 2235.117037] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2235.117226] env[62506]: INFO nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2235.117459] env[62506]: DEBUG oslo.service.loopingcall [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2235.117673] env[62506]: DEBUG nova.compute.manager [-] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2235.117770] env[62506]: DEBUG nova.network.neutron [-] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2235.130598] env[62506]: DEBUG nova.network.neutron [-] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2235.160780] env[62506]: DEBUG nova.compute.manager [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Received event network-changed-fa808efa-bc76-4bfb-a71f-aa17ce0b4862 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2235.160985] env[62506]: DEBUG nova.compute.manager [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Refreshing instance network info cache due to event network-changed-fa808efa-bc76-4bfb-a71f-aa17ce0b4862. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2235.161327] env[62506]: DEBUG oslo_concurrency.lockutils [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] Acquiring lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2235.161441] env[62506]: DEBUG oslo_concurrency.lockutils [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] Acquired lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2235.161604] env[62506]: DEBUG nova.network.neutron [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Refreshing network info cache for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2235.242333] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.830s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2235.242333] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2235.244564] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.798s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2235.245955] env[62506]: INFO nova.compute.claims [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2235.633025] env[62506]: DEBUG nova.network.neutron [-] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2235.681676] env[62506]: DEBUG nova.network.neutron [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2235.750274] env[62506]: DEBUG nova.compute.utils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2235.753491] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2235.753676] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2235.787397] env[62506]: DEBUG nova.network.neutron [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2235.804791] env[62506]: DEBUG nova.policy [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3c9c372663b34fcaa0dac3e1c657e044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fc10dc6a19fb41cdbf046f6da6dc69fe', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2235.874631] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2235.874970] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-58654322-e89c-47b5-ae13-44c2db4265af {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2235.881962] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2235.881962] env[62506]: value = "task-2190904" [ 2235.881962] env[62506]: _type = "Task" [ 2235.881962] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2235.889666] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190904, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2236.105067] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Successfully created port: 103c9a54-4f65-4ca2-b50f-329c69873c46 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2236.136769] env[62506]: INFO nova.compute.manager [-] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Took 1.02 seconds to deallocate network for instance. [ 2236.140106] env[62506]: DEBUG nova.compute.claims [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2236.140106] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2236.256791] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2236.290117] env[62506]: DEBUG oslo_concurrency.lockutils [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] Releasing lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2236.290117] env[62506]: DEBUG nova.compute.manager [req-bf4ebc3d-c4ce-49f7-bf20-7a51eccc19d2 req-94fc1363-3107-489a-9138-5825f0ddefd9 service nova] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Received event network-vif-deleted-fa808efa-bc76-4bfb-a71f-aa17ce0b4862 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2236.359771] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a37bbc51-2a2a-4a2d-8742-915882993d76 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.369218] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c05a40e5-4871-448d-a4b2-c996ab31cc20 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.404710] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a354145-48b3-420e-b336-f6558976831a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.412325] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190904, 'name': PowerOffVM_Task, 'duration_secs': 0.162505} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2236.414422] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2236.414664] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2236.415429] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4264ef80-4b1f-4d6d-8bb5-7ae7fa49a9cb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.418878] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa25db2f-3a10-465a-854e-31cc39a4b3a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.432483] env[62506]: DEBUG nova.compute.provider_tree [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2236.437381] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2236.437381] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-f8b66794-40e5-417a-9f3d-6554684bbc7c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.459831] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2236.460145] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2236.460351] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Deleting the datastore file [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2236.460608] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d52ebb93-fe21-4d04-820d-9cb15a8fbd4e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2236.466557] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2236.466557] env[62506]: value = "task-2190906" [ 2236.466557] env[62506]: _type = "Task" [ 2236.466557] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2236.474205] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190906, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2236.937195] env[62506]: DEBUG nova.scheduler.client.report [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2236.953884] env[62506]: ERROR nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2236.953884] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2236.953884] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2236.953884] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2236.953884] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2236.953884] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2236.953884] env[62506]: ERROR nova.compute.manager raise self.value [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2236.953884] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2236.953884] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2236.953884] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2236.954488] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2236.954488] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2236.954488] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2236.954488] env[62506]: ERROR nova.compute.manager [ 2236.954488] env[62506]: Traceback (most recent call last): [ 2236.954488] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2236.954488] env[62506]: listener.cb(fileno) [ 2236.954488] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2236.954488] env[62506]: result = function(*args, **kwargs) [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2236.954488] env[62506]: return func(*args, **kwargs) [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2236.954488] env[62506]: raise e [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2236.954488] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2236.954488] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2236.954488] env[62506]: with excutils.save_and_reraise_exception(): [ 2236.954488] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2236.954488] env[62506]: self.force_reraise() [ 2236.954488] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2236.954488] env[62506]: raise self.value [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2236.954488] env[62506]: updated_port = self._update_port( [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2236.954488] env[62506]: _ensure_no_port_binding_failure(port) [ 2236.954488] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2236.954488] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2236.955526] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2236.955526] env[62506]: Removing descriptor: 15 [ 2236.976323] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190906, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.08745} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2236.976565] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2236.976748] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2236.976930] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2237.186090] env[62506]: DEBUG nova.compute.manager [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Received event network-changed-103c9a54-4f65-4ca2-b50f-329c69873c46 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2237.186304] env[62506]: DEBUG nova.compute.manager [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Refreshing instance network info cache due to event network-changed-103c9a54-4f65-4ca2-b50f-329c69873c46. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2237.186528] env[62506]: DEBUG oslo_concurrency.lockutils [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] Acquiring lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2237.186675] env[62506]: DEBUG oslo_concurrency.lockutils [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] Acquired lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2237.186838] env[62506]: DEBUG nova.network.neutron [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Refreshing network info cache for port 103c9a54-4f65-4ca2-b50f-329c69873c46 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2237.268167] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2237.292176] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2237.292424] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2237.292585] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2237.292766] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2237.292915] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2237.293076] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2237.293289] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2237.293448] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2237.293611] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2237.293800] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2237.293978] env[62506]: DEBUG nova.virt.hardware [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2237.294852] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-376d5aae-aeec-49a1-851c-a807b14ffe5d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2237.302616] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c3dea2e-296c-4e56-a911-e92027281cb3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2237.315786] env[62506]: ERROR nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Traceback (most recent call last): [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] yield resources [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self.driver.spawn(context, instance, image_meta, [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] vm_ref = self.build_virtual_machine(instance, [ 2237.315786] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] vif_infos = vmwarevif.get_vif_info(self._session, [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] for vif in network_info: [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] return self._sync_wrapper(fn, *args, **kwargs) [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self.wait() [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self[:] = self._gt.wait() [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] return self._exit_event.wait() [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2237.316236] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] current.throw(*self._exc) [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] result = function(*args, **kwargs) [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] return func(*args, **kwargs) [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] raise e [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] nwinfo = self.network_api.allocate_for_instance( [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] created_port_ids = self._update_ports_for_instance( [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] with excutils.save_and_reraise_exception(): [ 2237.316703] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self.force_reraise() [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] raise self.value [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] updated_port = self._update_port( [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] _ensure_no_port_binding_failure(port) [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] raise exception.PortBindingFailed(port_id=port['id']) [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2237.317401] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] [ 2237.317401] env[62506]: INFO nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Terminating instance [ 2237.442118] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.197s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2237.442850] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2237.449328] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.015s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2237.451558] env[62506]: INFO nova.compute.claims [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2237.707391] env[62506]: DEBUG nova.network.neutron [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2237.779067] env[62506]: DEBUG nova.network.neutron [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2237.820431] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2237.957424] env[62506]: DEBUG nova.compute.utils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2237.961280] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2237.961505] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2238.004602] env[62506]: DEBUG nova.policy [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e508d2263c0c47d5bccfb1e2f1409316', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7d771ac74ac54fb38a84337eed72348a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2238.011292] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2238.011519] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2238.011702] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2238.011893] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2238.012052] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2238.012203] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2238.012407] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2238.012564] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2238.012728] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2238.012890] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2238.013079] env[62506]: DEBUG nova.virt.hardware [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2238.013930] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8ceb1b9-3325-43f2-9ce6-e0deb00fc68c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.021475] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36805188-1850-4780-a796-4295561c5a1e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.035065] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance VIF info [] {{(pid=62506) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2238.040659] env[62506]: DEBUG oslo.service.loopingcall [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2238.041144] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Creating VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 2238.041472] env[62506]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9decc7ab-c50b-4cbd-85fe-c5717105dab8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.058893] env[62506]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2238.058893] env[62506]: value = "task-2190907" [ 2238.058893] env[62506]: _type = "Task" [ 2238.058893] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2238.066358] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190907, 'name': CreateVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2238.281960] env[62506]: DEBUG oslo_concurrency.lockutils [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] Releasing lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2238.282793] env[62506]: DEBUG nova.compute.manager [req-efb5dcc8-948b-4f53-b9b3-3853f20086b6 req-1343b820-45ae-4e57-b466-ba1d4c75b7a2 service nova] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Received event network-vif-deleted-103c9a54-4f65-4ca2-b50f-329c69873c46 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2238.283228] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2238.283416] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2238.301250] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Successfully created port: ed6f4894-46f0-4939-a3c2-e756794f1b5b {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2238.462290] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2238.564450] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34b2d81f-f943-4af9-a175-a182cbcecaa8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.572527] env[62506]: DEBUG oslo_vmware.api [-] Task: {'id': task-2190907, 'name': CreateVM_Task, 'duration_secs': 0.252163} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2238.574496] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Created VM on the ESX host {{(pid=62506) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 2238.574969] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2238.575151] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2238.575460] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2238.576467] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6337ca80-80c3-4459-997a-daec81e196ad {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.579268] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8c06a762-7820-4e26-b499-d2df83830aae {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.608092] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2238.608092] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52b6a6c5-b723-cd77-d3e9-d1a1b64d25ca" [ 2238.608092] env[62506]: _type = "Task" [ 2238.608092] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2238.608984] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b52a006c-07f4-4b32-9c30-1a104591184b {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.621729] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29dd0aef-0b77-46b9-99a0-ae23b2cfe47e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.625292] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52b6a6c5-b723-cd77-d3e9-d1a1b64d25ca, 'name': SearchDatastore_Task, 'duration_secs': 0.008919} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2238.625554] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2238.625894] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Processing image 580d9529-f92b-46b3-a9d4-08013a8922b3 {{(pid=62506) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2238.626046] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2238.627406] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2238.627406] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2238.627406] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-944b401b-e5b5-40cb-ad2b-2e536d2e72af {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.636221] env[62506]: DEBUG nova.compute.provider_tree [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2238.643831] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62506) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2238.643831] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62506) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 2238.644019] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d1089196-7a8b-4874-9d06-276e175c54c6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2238.649189] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2238.649189] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]52c8a0a0-775d-c350-b2e5-94b8b4820026" [ 2238.649189] env[62506]: _type = "Task" [ 2238.649189] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2238.658309] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52c8a0a0-775d-c350-b2e5-94b8b4820026, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2238.802913] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2238.909177] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2239.142581] env[62506]: DEBUG nova.scheduler.client.report [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2239.159922] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]52c8a0a0-775d-c350-b2e5-94b8b4820026, 'name': SearchDatastore_Task, 'duration_secs': 0.008431} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2239.160770] env[62506]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b3da9c41-a110-4b49-9a98-3ea8daa51268 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.166014] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2239.166014] env[62506]: value = "session[522f1033-663c-6371-4da3-e352dc269c93]529d88a7-7132-3176-57b5-478750cf67de" [ 2239.166014] env[62506]: _type = "Task" [ 2239.166014] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2239.173430] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]529d88a7-7132-3176-57b5-478750cf67de, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2239.216955] env[62506]: DEBUG nova.compute.manager [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Received event network-changed-ed6f4894-46f0-4939-a3c2-e756794f1b5b {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2239.217167] env[62506]: DEBUG nova.compute.manager [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Refreshing instance network info cache due to event network-changed-ed6f4894-46f0-4939-a3c2-e756794f1b5b. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2239.217377] env[62506]: DEBUG oslo_concurrency.lockutils [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] Acquiring lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2239.217518] env[62506]: DEBUG oslo_concurrency.lockutils [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] Acquired lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2239.217775] env[62506]: DEBUG nova.network.neutron [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Refreshing network info cache for port ed6f4894-46f0-4939-a3c2-e756794f1b5b {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2239.227096] env[62506]: ERROR nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2239.227096] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2239.227096] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2239.227096] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2239.227096] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2239.227096] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2239.227096] env[62506]: ERROR nova.compute.manager raise self.value [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2239.227096] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2239.227096] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2239.227096] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2239.227787] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2239.227787] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2239.227787] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2239.227787] env[62506]: ERROR nova.compute.manager [ 2239.227787] env[62506]: Traceback (most recent call last): [ 2239.227787] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2239.227787] env[62506]: listener.cb(fileno) [ 2239.227787] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2239.227787] env[62506]: result = function(*args, **kwargs) [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2239.227787] env[62506]: return func(*args, **kwargs) [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2239.227787] env[62506]: raise e [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2239.227787] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2239.227787] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2239.227787] env[62506]: with excutils.save_and_reraise_exception(): [ 2239.227787] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2239.227787] env[62506]: self.force_reraise() [ 2239.227787] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2239.227787] env[62506]: raise self.value [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2239.227787] env[62506]: updated_port = self._update_port( [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2239.227787] env[62506]: _ensure_no_port_binding_failure(port) [ 2239.227787] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2239.227787] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2239.228899] env[62506]: nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2239.228899] env[62506]: Removing descriptor: 15 [ 2239.412059] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2239.412517] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2239.412718] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2239.413023] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3e5bcf75-31d4-480d-80fe-a8dd3299e805 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.422466] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aba742cf-5ce7-48a4-916a-2084d67e63c9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.444491] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance df6be48d-bb76-4efd-9da5-3e6ce6467667 could not be found. [ 2239.444731] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2239.444937] env[62506]: INFO nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2239.445196] env[62506]: DEBUG oslo.service.loopingcall [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2239.445427] env[62506]: DEBUG nova.compute.manager [-] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2239.445523] env[62506]: DEBUG nova.network.neutron [-] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2239.467618] env[62506]: DEBUG nova.network.neutron [-] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2239.474378] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2239.499165] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2239.499416] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2239.499573] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2239.499754] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2239.499899] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2239.500053] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2239.500263] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2239.500420] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2239.500587] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2239.500746] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2239.500916] env[62506]: DEBUG nova.virt.hardware [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2239.501804] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e10d7e09-c6b3-4d62-9907-6d24eb632a89 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.509335] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dde90fce-0a01-4cec-8908-d7a9bd277d50 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.526892] env[62506]: ERROR nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Traceback (most recent call last): [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] yield resources [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self.driver.spawn(context, instance, image_meta, [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] vm_ref = self.build_virtual_machine(instance, [ 2239.526892] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] vif_infos = vmwarevif.get_vif_info(self._session, [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] for vif in network_info: [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] return self._sync_wrapper(fn, *args, **kwargs) [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self.wait() [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self[:] = self._gt.wait() [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] return self._exit_event.wait() [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2239.527486] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] current.throw(*self._exc) [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] result = function(*args, **kwargs) [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] return func(*args, **kwargs) [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] raise e [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] nwinfo = self.network_api.allocate_for_instance( [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] created_port_ids = self._update_ports_for_instance( [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] with excutils.save_and_reraise_exception(): [ 2239.527994] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self.force_reraise() [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] raise self.value [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] updated_port = self._update_port( [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] _ensure_no_port_binding_failure(port) [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] raise exception.PortBindingFailed(port_id=port['id']) [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2239.528687] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] [ 2239.528687] env[62506]: INFO nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Terminating instance [ 2239.647868] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.199s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2239.648404] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2239.651013] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 7.537s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2239.651197] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2239.651347] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2239.651622] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 5.294s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2239.655432] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08104eca-188c-4be4-bfe0-5d25271a0ed7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.663966] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31a2ea70-61e6-400a-800d-0982f1d432d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.676157] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': session[522f1033-663c-6371-4da3-e352dc269c93]529d88a7-7132-3176-57b5-478750cf67de, 'name': SearchDatastore_Task, 'duration_secs': 0.009706} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2239.683895] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Releasing lock "[datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2239.684184] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 2239.684549] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1948ead1-db71-4deb-b52d-2e000696b41c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.686787] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9356acea-c755-4672-873d-b6bfa3ced800 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.694271] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c0178d9-0c12-4d9b-a049-b19ebe42dcd4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2239.698129] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2239.698129] env[62506]: value = "task-2190908" [ 2239.698129] env[62506]: _type = "Task" [ 2239.698129] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2239.728088] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181251MB free_disk=176GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2239.728266] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2239.733639] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190908, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2239.752625] env[62506]: DEBUG nova.network.neutron [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2239.830664] env[62506]: DEBUG nova.network.neutron [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2239.970696] env[62506]: DEBUG nova.network.neutron [-] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2240.031720] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2240.156779] env[62506]: DEBUG nova.compute.utils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2240.158247] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2240.158440] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2240.209646] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190908, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.472263} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2240.211058] env[62506]: DEBUG nova.policy [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b6a7dad8dbbb4c17bbaaa1ff4a904b03', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'edda71ca67d247b38a02045748dbb728', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2240.212666] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/580d9529-f92b-46b3-a9d4-08013a8922b3/580d9529-f92b-46b3-a9d4-08013a8922b3.vmdk to [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk {{(pid=62506) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 2240.212952] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Extending root virtual disk to 1048576 {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 2240.213192] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-52655936-4817-4fb0-8a98-922ffbda6f8e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.222605] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2240.222605] env[62506]: value = "task-2190909" [ 2240.222605] env[62506]: _type = "Task" [ 2240.222605] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2240.233510] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190909, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2240.264564] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e4b40e2-6412-4897-91be-e2b4c0b721fc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.271620] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d591a77-b311-4257-a851-432c258f6cd8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.304101] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e673735-8b79-4279-83f3-52f083f5c5e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.312242] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c7c5b2b-432f-42c5-8df7-2a62c53e36f4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.325708] env[62506]: DEBUG nova.compute.provider_tree [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2240.332841] env[62506]: DEBUG oslo_concurrency.lockutils [req-944d246a-1dbe-475c-9dd5-924af9e9edcc req-c0fc1633-7293-47b5-9474-298c99366752 service nova] Releasing lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2240.333280] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2240.333519] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2240.474447] env[62506]: INFO nova.compute.manager [-] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Took 1.03 seconds to deallocate network for instance. [ 2240.476689] env[62506]: DEBUG nova.compute.claims [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2240.476890] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2240.661561] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2240.703589] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Successfully created port: 9ae8ed80-25f5-4d9e-8212-eba4ab116a94 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2240.732457] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190909, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.065632} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2240.732716] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Extended root virtual disk {{(pid=62506) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 2240.733461] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f759437-f717-4853-a9cf-08a582e64a9e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.753726] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Reconfiguring VM instance instance-0000005c to attach disk [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 2240.753963] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-6abbc0c2-c4c8-4cac-bf0f-56286a786639 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2240.775386] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2240.775386] env[62506]: value = "task-2190910" [ 2240.775386] env[62506]: _type = "Task" [ 2240.775386] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2240.783754] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190910, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2240.828969] env[62506]: DEBUG nova.scheduler.client.report [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2240.855259] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2240.970929] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2241.245599] env[62506]: DEBUG nova.compute.manager [req-4109a49a-ac14-4e01-a398-db7d1270c80b req-2ec76add-d39f-45d1-982e-53c92b98a339 service nova] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Received event network-vif-deleted-ed6f4894-46f0-4939-a3c2-e756794f1b5b {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2241.285055] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190910, 'name': ReconfigVM_Task, 'duration_secs': 0.262597} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2241.285344] env[62506]: DEBUG nova.virt.vmwareapi.volumeops [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Reconfigured VM instance instance-0000005c to attach disk [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d/e6f9199d-16c7-4d18-87a4-723be0eb3a2d.vmdk or device None with type sparse {{(pid=62506) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 2241.285932] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-d6d14580-84fe-4124-802f-9c5fec0532aa {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2241.292416] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2241.292416] env[62506]: value = "task-2190911" [ 2241.292416] env[62506]: _type = "Task" [ 2241.292416] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2241.301369] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190911, 'name': Rename_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2241.337216] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.685s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2241.337947] env[62506]: ERROR nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Traceback (most recent call last): [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self.driver.spawn(context, instance, image_meta, [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] vm_ref = self.build_virtual_machine(instance, [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] vif_infos = vmwarevif.get_vif_info(self._session, [ 2241.337947] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] for vif in network_info: [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] return self._sync_wrapper(fn, *args, **kwargs) [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self.wait() [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self[:] = self._gt.wait() [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] return self._exit_event.wait() [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] current.throw(*self._exc) [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2241.338378] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] result = function(*args, **kwargs) [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] return func(*args, **kwargs) [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] raise e [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] nwinfo = self.network_api.allocate_for_instance( [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] created_port_ids = self._update_ports_for_instance( [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] with excutils.save_and_reraise_exception(): [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] self.force_reraise() [ 2241.338757] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] raise self.value [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] updated_port = self._update_port( [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] _ensure_no_port_binding_failure(port) [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] raise exception.PortBindingFailed(port_id=port['id']) [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] nova.exception.PortBindingFailed: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. [ 2241.339136] env[62506]: ERROR nova.compute.manager [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] [ 2241.339136] env[62506]: DEBUG nova.compute.utils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2241.339894] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.373s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2241.341717] env[62506]: INFO nova.compute.claims [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2241.344414] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Build of instance 9bede4a7-88ad-4362-afd8-3cfbb71afecd was re-scheduled: Binding failed for port 59852802-810f-4bab-8fd4-f3cda8878d02, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2241.344840] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2241.345073] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquiring lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2241.345320] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Acquired lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2241.345400] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2241.473548] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2241.474077] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2241.474300] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2241.474610] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4bb2d910-14de-4a9e-9f3a-16cf386628d8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2241.486983] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d655b7e4-76fc-4af1-8de6-536da7401f74 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2241.507740] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0 could not be found. [ 2241.508496] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2241.508496] env[62506]: INFO nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2241.508496] env[62506]: DEBUG oslo.service.loopingcall [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2241.508679] env[62506]: DEBUG nova.compute.manager [-] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2241.508679] env[62506]: DEBUG nova.network.neutron [-] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2241.524984] env[62506]: DEBUG nova.network.neutron [-] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2241.662280] env[62506]: ERROR nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2241.662280] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2241.662280] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2241.662280] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2241.662280] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2241.662280] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2241.662280] env[62506]: ERROR nova.compute.manager raise self.value [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2241.662280] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2241.662280] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2241.662280] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2241.662837] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2241.662837] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2241.662837] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2241.662837] env[62506]: ERROR nova.compute.manager [ 2241.662837] env[62506]: Traceback (most recent call last): [ 2241.662837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2241.662837] env[62506]: listener.cb(fileno) [ 2241.662837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2241.662837] env[62506]: result = function(*args, **kwargs) [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2241.662837] env[62506]: return func(*args, **kwargs) [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2241.662837] env[62506]: raise e [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2241.662837] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2241.662837] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2241.662837] env[62506]: with excutils.save_and_reraise_exception(): [ 2241.662837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2241.662837] env[62506]: self.force_reraise() [ 2241.662837] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2241.662837] env[62506]: raise self.value [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2241.662837] env[62506]: updated_port = self._update_port( [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2241.662837] env[62506]: _ensure_no_port_binding_failure(port) [ 2241.662837] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2241.662837] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2241.663856] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2241.663856] env[62506]: Removing descriptor: 15 [ 2241.671174] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2241.698598] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2241.698839] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2241.698999] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2241.699198] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2241.699345] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2241.699488] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2241.699695] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2241.699857] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2241.700032] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2241.700203] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2241.700376] env[62506]: DEBUG nova.virt.hardware [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2241.701226] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49156c02-fda0-4cf3-a92a-22891b314cea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2241.709006] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-246dbe38-6b3b-4736-b866-7d74218baeba {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2241.722482] env[62506]: ERROR nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Traceback (most recent call last): [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] yield resources [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self.driver.spawn(context, instance, image_meta, [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] vm_ref = self.build_virtual_machine(instance, [ 2241.722482] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] vif_infos = vmwarevif.get_vif_info(self._session, [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] for vif in network_info: [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] return self._sync_wrapper(fn, *args, **kwargs) [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self.wait() [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self[:] = self._gt.wait() [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] return self._exit_event.wait() [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2241.722943] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] current.throw(*self._exc) [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] result = function(*args, **kwargs) [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] return func(*args, **kwargs) [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] raise e [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] nwinfo = self.network_api.allocate_for_instance( [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] created_port_ids = self._update_ports_for_instance( [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] with excutils.save_and_reraise_exception(): [ 2241.723409] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self.force_reraise() [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] raise self.value [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] updated_port = self._update_port( [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] _ensure_no_port_binding_failure(port) [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] raise exception.PortBindingFailed(port_id=port['id']) [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2241.723890] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] [ 2241.723890] env[62506]: INFO nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Terminating instance [ 2241.802921] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190911, 'name': Rename_Task, 'duration_secs': 0.155498} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2241.803234] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powering on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 2241.803491] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-27c95ba9-c128-483f-8476-af7d2ae7ff19 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2241.809694] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2241.809694] env[62506]: value = "task-2190912" [ 2241.809694] env[62506]: _type = "Task" [ 2241.809694] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2241.817163] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190912, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2241.878869] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2241.961466] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2242.027112] env[62506]: DEBUG nova.network.neutron [-] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2242.226494] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquiring lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2242.226682] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquired lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2242.226870] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2242.320795] env[62506]: DEBUG oslo_vmware.api [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190912, 'name': PowerOnVM_Task, 'duration_secs': 0.417243} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2242.321171] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powered on the VM {{(pid=62506) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 2242.321495] env[62506]: DEBUG nova.compute.manager [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Checking state {{(pid=62506) _get_power_state /opt/stack/nova/nova/compute/manager.py:1798}} [ 2242.322364] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88a173db-1e37-4d85-a0ac-53c00b5c8363 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2242.438639] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf862c52-cc3e-40e4-8ce4-fccfd6ee6127 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2242.445875] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a03ec30-eb38-46f7-bf58-e49745f8d870 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2242.474116] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Releasing lock "refresh_cache-9bede4a7-88ad-4362-afd8-3cfbb71afecd" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2242.474382] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2242.474570] env[62506]: DEBUG nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2242.474760] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2242.477031] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ede5c8a-a8cf-40ab-8f80-8327decae7b7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2242.485027] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26030b09-d1af-4553-ba97-696bd37b0638 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2242.498583] env[62506]: DEBUG nova.compute.provider_tree [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2242.500461] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2242.529993] env[62506]: INFO nova.compute.manager [-] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Took 1.02 seconds to deallocate network for instance. [ 2242.532111] env[62506]: DEBUG nova.compute.claims [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2242.532315] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2242.743843] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2242.814268] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2242.836541] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2243.003244] env[62506]: DEBUG nova.scheduler.client.report [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2243.006921] env[62506]: DEBUG nova.network.neutron [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2243.269820] env[62506]: DEBUG nova.compute.manager [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Received event network-changed-9ae8ed80-25f5-4d9e-8212-eba4ab116a94 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2243.269992] env[62506]: DEBUG nova.compute.manager [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Refreshing instance network info cache due to event network-changed-9ae8ed80-25f5-4d9e-8212-eba4ab116a94. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2243.270209] env[62506]: DEBUG oslo_concurrency.lockutils [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] Acquiring lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2243.317226] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Releasing lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2243.317666] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2243.317829] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2243.318139] env[62506]: DEBUG oslo_concurrency.lockutils [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] Acquired lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2243.318310] env[62506]: DEBUG nova.network.neutron [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Refreshing network info cache for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2243.319337] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d8ec0d81-302a-4e25-a8c4-a9c9a2ce2402 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2243.329021] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7861aa7b-b0a2-418a-a281-0cab00444f8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2243.351408] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 54ea1f68-8c18-4564-9743-6aec7d1995e9 could not be found. [ 2243.351626] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2243.351890] env[62506]: INFO nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2243.352155] env[62506]: DEBUG oslo.service.loopingcall [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2243.352391] env[62506]: DEBUG nova.compute.manager [-] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2243.352490] env[62506]: DEBUG nova.network.neutron [-] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2243.367367] env[62506]: DEBUG nova.network.neutron [-] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2243.509506] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.169s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2243.510051] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2243.512733] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.373s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2243.515848] env[62506]: INFO nova.compute.manager [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] [instance: 9bede4a7-88ad-4362-afd8-3cfbb71afecd] Took 1.04 seconds to deallocate network for instance. [ 2243.518077] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2243.518308] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2243.518506] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2243.518688] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2243.518854] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2243.520484] env[62506]: INFO nova.compute.manager [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Terminating instance [ 2243.837232] env[62506]: DEBUG nova.network.neutron [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2243.869779] env[62506]: DEBUG nova.network.neutron [-] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2243.907443] env[62506]: DEBUG nova.network.neutron [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2244.020188] env[62506]: DEBUG nova.compute.utils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2244.025228] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2244.025680] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2244.030114] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "refresh_cache-e6f9199d-16c7-4d18-87a4-723be0eb3a2d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2244.030737] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquired lock "refresh_cache-e6f9199d-16c7-4d18-87a4-723be0eb3a2d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2244.030737] env[62506]: DEBUG nova.network.neutron [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2244.068770] env[62506]: DEBUG nova.policy [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce3d50af36d14790baee7ee06f0debad', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3e75732654f946cfad5c9bf0242d3b7c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2244.117365] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99f6dbad-d1d5-4e67-989b-778fd61d3ddc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2244.124631] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04983896-87e2-4a01-8ce2-2baebef136d3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2244.154149] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4ac9d00-0f8f-47ba-9444-6798d09e65c3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2244.161331] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29b507b0-d94c-48ca-9298-69a5eb7a69e0 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2244.174283] env[62506]: DEBUG nova.compute.provider_tree [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2244.357068] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Successfully created port: 767641d2-8f80-42ab-bd32-3820ef307258 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2244.373672] env[62506]: INFO nova.compute.manager [-] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Took 1.02 seconds to deallocate network for instance. [ 2244.375953] env[62506]: DEBUG nova.compute.claims [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2244.376142] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2244.409793] env[62506]: DEBUG oslo_concurrency.lockutils [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] Releasing lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2244.410114] env[62506]: DEBUG nova.compute.manager [req-cd3ea484-e21e-4e5e-aa52-ed5a4d809d5b req-41e1d111-df9e-44b8-9429-29cb0fb68f87 service nova] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Received event network-vif-deleted-9ae8ed80-25f5-4d9e-8212-eba4ab116a94 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2244.527041] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2244.557567] env[62506]: INFO nova.scheduler.client.report [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Deleted allocations for instance 9bede4a7-88ad-4362-afd8-3cfbb71afecd [ 2244.564398] env[62506]: DEBUG nova.network.neutron [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2244.663794] env[62506]: DEBUG nova.network.neutron [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2244.677634] env[62506]: DEBUG nova.scheduler.client.report [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 176, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2245.070529] env[62506]: DEBUG oslo_concurrency.lockutils [None req-beede18d-8163-4c4e-8988-6b80c9cc9df0 tempest-ServerActionsTestJSON-1594932941 tempest-ServerActionsTestJSON-1594932941-project-member] Lock "9bede4a7-88ad-4362-afd8-3cfbb71afecd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 26.659s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2245.166156] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Releasing lock "refresh_cache-e6f9199d-16c7-4d18-87a4-723be0eb3a2d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2245.166604] env[62506]: DEBUG nova.compute.manager [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2245.166804] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2245.167719] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac55d8cd-a37b-4f3e-add9-3aea1f17ef4e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2245.176254] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powering off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 2245.176509] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-95f28bb2-0600-40f3-8aad-d8192f3977a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2245.183428] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.671s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2245.184109] env[62506]: ERROR nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Traceback (most recent call last): [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self.driver.spawn(context, instance, image_meta, [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] vm_ref = self.build_virtual_machine(instance, [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] vif_infos = vmwarevif.get_vif_info(self._session, [ 2245.184109] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] for vif in network_info: [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] return self._sync_wrapper(fn, *args, **kwargs) [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self.wait() [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self[:] = self._gt.wait() [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] return self._exit_event.wait() [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] current.throw(*self._exc) [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2245.184789] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] result = function(*args, **kwargs) [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] return func(*args, **kwargs) [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] raise e [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] nwinfo = self.network_api.allocate_for_instance( [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] created_port_ids = self._update_ports_for_instance( [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] with excutils.save_and_reraise_exception(): [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] self.force_reraise() [ 2245.185349] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] raise self.value [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] updated_port = self._update_port( [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] _ensure_no_port_binding_failure(port) [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] raise exception.PortBindingFailed(port_id=port['id']) [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] nova.exception.PortBindingFailed: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. [ 2245.186041] env[62506]: ERROR nova.compute.manager [instance: 38651166-97e3-467b-b4be-7002ecbd6396] [ 2245.186041] env[62506]: DEBUG nova.compute.utils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2245.186041] env[62506]: DEBUG oslo_vmware.api [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2245.186661] env[62506]: value = "task-2190913" [ 2245.186661] env[62506]: _type = "Task" [ 2245.186661] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2245.186661] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Build of instance 38651166-97e3-467b-b4be-7002ecbd6396 was re-scheduled: Binding failed for port fa808efa-bc76-4bfb-a71f-aa17ce0b4862, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2245.186876] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2245.187063] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2245.187215] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquired lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2245.187373] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2245.188385] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 5.460s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2245.202180] env[62506]: DEBUG oslo_vmware.api [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190913, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2245.305368] env[62506]: DEBUG nova.compute.manager [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Received event network-changed-767641d2-8f80-42ab-bd32-3820ef307258 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2245.307771] env[62506]: DEBUG nova.compute.manager [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Refreshing instance network info cache due to event network-changed-767641d2-8f80-42ab-bd32-3820ef307258. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2245.307771] env[62506]: DEBUG oslo_concurrency.lockutils [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] Acquiring lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2245.307771] env[62506]: DEBUG oslo_concurrency.lockutils [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] Acquired lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2245.307771] env[62506]: DEBUG nova.network.neutron [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Refreshing network info cache for port 767641d2-8f80-42ab-bd32-3820ef307258 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2245.384841] env[62506]: ERROR nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2245.384841] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2245.384841] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2245.384841] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2245.384841] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2245.384841] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2245.384841] env[62506]: ERROR nova.compute.manager raise self.value [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2245.384841] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2245.384841] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2245.384841] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2245.385733] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2245.385733] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2245.385733] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2245.385733] env[62506]: ERROR nova.compute.manager [ 2245.385733] env[62506]: Traceback (most recent call last): [ 2245.385733] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2245.385733] env[62506]: listener.cb(fileno) [ 2245.385733] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2245.385733] env[62506]: result = function(*args, **kwargs) [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2245.385733] env[62506]: return func(*args, **kwargs) [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2245.385733] env[62506]: raise e [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2245.385733] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2245.385733] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2245.385733] env[62506]: with excutils.save_and_reraise_exception(): [ 2245.385733] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2245.385733] env[62506]: self.force_reraise() [ 2245.385733] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2245.385733] env[62506]: raise self.value [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2245.385733] env[62506]: updated_port = self._update_port( [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2245.385733] env[62506]: _ensure_no_port_binding_failure(port) [ 2245.385733] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2245.385733] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2245.386606] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2245.386606] env[62506]: Removing descriptor: 15 [ 2245.536402] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2245.561636] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2245.561869] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2245.562039] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2245.562226] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2245.562369] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2245.562512] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2245.562713] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2245.562867] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2245.563038] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2245.563201] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2245.563373] env[62506]: DEBUG nova.virt.hardware [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2245.564271] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da549149-3ae3-48be-8c26-5a5cd2a5b336 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2245.572378] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7181c755-0eed-4d23-9a05-10d9d38c513a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2245.586136] env[62506]: ERROR nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Traceback (most recent call last): [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] yield resources [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self.driver.spawn(context, instance, image_meta, [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] vm_ref = self.build_virtual_machine(instance, [ 2245.586136] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] vif_infos = vmwarevif.get_vif_info(self._session, [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] for vif in network_info: [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] return self._sync_wrapper(fn, *args, **kwargs) [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self.wait() [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self[:] = self._gt.wait() [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] return self._exit_event.wait() [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2245.586881] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] current.throw(*self._exc) [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] result = function(*args, **kwargs) [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] return func(*args, **kwargs) [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] raise e [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] nwinfo = self.network_api.allocate_for_instance( [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] created_port_ids = self._update_ports_for_instance( [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] with excutils.save_and_reraise_exception(): [ 2245.587496] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self.force_reraise() [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] raise self.value [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] updated_port = self._update_port( [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] _ensure_no_port_binding_failure(port) [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] raise exception.PortBindingFailed(port_id=port['id']) [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2245.588078] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] [ 2245.588078] env[62506]: INFO nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Terminating instance [ 2245.705059] env[62506]: DEBUG oslo_vmware.api [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190913, 'name': PowerOffVM_Task, 'duration_secs': 0.127222} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2245.705191] env[62506]: DEBUG nova.virt.vmwareapi.vm_util [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Powered off the VM {{(pid=62506) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 2245.705281] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Unregistering the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 2245.705529] env[62506]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a73aa7ce-55d7-4099-9437-d446e2908043 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2245.710725] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2245.730685] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Unregistered the VM {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 2245.730948] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Deleting contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 2245.731154] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Deleting the datastore file [datastore2] e6f9199d-16c7-4d18-87a4-723be0eb3a2d {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2245.731405] env[62506]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-e05a9882-0b25-4142-801e-b29af44cf73f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2245.738062] env[62506]: DEBUG oslo_vmware.api [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for the task: (returnval){ [ 2245.738062] env[62506]: value = "task-2190915" [ 2245.738062] env[62506]: _type = "Task" [ 2245.738062] env[62506]: } to complete. {{(pid=62506) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2245.746425] env[62506]: DEBUG oslo_vmware.api [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190915, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2245.783578] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2245.826430] env[62506]: DEBUG nova.network.neutron [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2245.898251] env[62506]: DEBUG nova.network.neutron [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2246.090624] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquiring lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2246.223337] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance e6f9199d-16c7-4d18-87a4-723be0eb3a2d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.248924] env[62506]: DEBUG oslo_vmware.api [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Task: {'id': task-2190915, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.093699} completed successfully. {{(pid=62506) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2246.249224] env[62506]: DEBUG nova.virt.vmwareapi.ds_util [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Deleted the datastore file {{(pid=62506) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2246.249413] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Deleted contents of the VM from datastore datastore2 {{(pid=62506) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 2246.250032] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2246.250032] env[62506]: INFO nova.compute.manager [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Took 1.08 seconds to destroy the instance on the hypervisor. [ 2246.250032] env[62506]: DEBUG oslo.service.loopingcall [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2246.250297] env[62506]: DEBUG nova.compute.manager [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2246.250395] env[62506]: DEBUG nova.network.neutron [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2246.279893] env[62506]: DEBUG nova.network.neutron [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2246.286064] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Releasing lock "refresh_cache-38651166-97e3-467b-b4be-7002ecbd6396" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2246.286331] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2246.286685] env[62506]: DEBUG nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2246.286685] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2246.305651] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2246.403342] env[62506]: DEBUG oslo_concurrency.lockutils [req-cc2e0a63-f884-419c-b79f-99a228fec5e5 req-c87c2049-8df4-4006-bfde-b4967d78fb3d service nova] Releasing lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2246.403342] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquired lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2246.403342] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2246.728521] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 38651166-97e3-467b-b4be-7002ecbd6396 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2246.728521] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance df6be48d-bb76-4efd-9da5-3e6ce6467667 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.728521] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.728521] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 54ea1f68-8c18-4564-9743-6aec7d1995e9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.728787] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Instance 2153994f-947b-47a5-8eca-f665c28eb87f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62506) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.728787] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2246.728787] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1472MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2246.782855] env[62506]: DEBUG nova.network.neutron [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2246.808715] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddaddf11-31e2-40b1-8eca-df0bef69a2f2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.812409] env[62506]: DEBUG nova.network.neutron [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2246.819432] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfff7145-5f09-487d-b215-1f1ad42a366e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.851269] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3964279d-abae-4cdf-b54d-26d01a651137 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.858456] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ae0fb42-3894-46f5-9123-ab9dfb924e76 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.871451] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2246.920419] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2246.998169] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2247.285747] env[62506]: INFO nova.compute.manager [-] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Took 1.04 seconds to deallocate network for instance. [ 2247.315393] env[62506]: INFO nova.compute.manager [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: 38651166-97e3-467b-b4be-7002ecbd6396] Took 1.03 seconds to deallocate network for instance. [ 2247.329744] env[62506]: DEBUG nova.compute.manager [req-b8180ce6-97d3-41b9-9822-edc6f061a2a6 req-97d08c6d-cfcb-41f0-9ff1-15783287c39b service nova] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Received event network-vif-deleted-767641d2-8f80-42ab-bd32-3820ef307258 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2247.401870] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updated inventory for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with generation 117 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:975}} [ 2247.402141] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating resource provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 generation from 117 to 118 during operation: update_inventory {{(pid=62506) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 2247.402296] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Updating inventory in ProviderTree for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2247.501106] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Releasing lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2247.501562] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2247.501788] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2247.502097] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3d4d189e-7027-4d82-97bd-f9c2bad095b7 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2247.511124] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da97e053-d1dc-4995-bcb0-d452cf2f1185 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2247.533231] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2153994f-947b-47a5-8eca-f665c28eb87f could not be found. [ 2247.533442] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2247.533622] env[62506]: INFO nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2247.533894] env[62506]: DEBUG oslo.service.loopingcall [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2247.534282] env[62506]: DEBUG nova.compute.manager [-] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2247.534282] env[62506]: DEBUG nova.network.neutron [-] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2247.549234] env[62506]: DEBUG nova.network.neutron [-] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2247.791820] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2247.907473] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2247.907693] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.719s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2247.907953] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.431s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2248.051511] env[62506]: DEBUG nova.network.neutron [-] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2248.342875] env[62506]: INFO nova.scheduler.client.report [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Deleted allocations for instance 38651166-97e3-467b-b4be-7002ecbd6396 [ 2248.478988] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dbdfb53-c9df-4bfa-8bf5-a7a16f78c031 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2248.486933] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f645ae3-c7a1-453d-af3c-41854042eec6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2248.519182] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28f62fb2-4f9f-4983-b0f6-8a801a390535 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2248.526638] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fca30746-07e7-4e5a-a23e-3d02588adce4 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2248.539763] env[62506]: DEBUG nova.compute.provider_tree [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2248.554471] env[62506]: INFO nova.compute.manager [-] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Took 1.02 seconds to deallocate network for instance. [ 2248.556485] env[62506]: DEBUG nova.compute.claims [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2248.556660] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2248.850246] env[62506]: DEBUG oslo_concurrency.lockutils [None req-34621ea9-420b-463d-ad0b-5b9e99642b3a tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "38651166-97e3-467b-b4be-7002ecbd6396" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 29.703s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2248.910913] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2248.911169] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2248.911317] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Starting heal instance info cache {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10445}} [ 2248.911438] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Rebuilding the list of instances to heal {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10449}} [ 2249.042549] env[62506]: DEBUG nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2249.415494] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2249.415657] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2249.415793] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Skipping network cache update for instance because it is Building. {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10458}} [ 2249.433547] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "refresh_cache-e6f9199d-16c7-4d18-87a4-723be0eb3a2d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2249.433547] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquired lock "refresh_cache-e6f9199d-16c7-4d18-87a4-723be0eb3a2d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2249.433547] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Forcefully refreshing network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2061}} [ 2249.434290] env[62506]: DEBUG nova.objects.instance [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lazy-loading 'info_cache' on Instance uuid e6f9199d-16c7-4d18-87a4-723be0eb3a2d {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 2249.547608] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.639s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2249.548285] env[62506]: ERROR nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Traceback (most recent call last): [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self.driver.spawn(context, instance, image_meta, [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] vm_ref = self.build_virtual_machine(instance, [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] vif_infos = vmwarevif.get_vif_info(self._session, [ 2249.548285] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] for vif in network_info: [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] return self._sync_wrapper(fn, *args, **kwargs) [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self.wait() [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self[:] = self._gt.wait() [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] return self._exit_event.wait() [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] current.throw(*self._exc) [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2249.548692] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] result = function(*args, **kwargs) [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] return func(*args, **kwargs) [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] raise e [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] nwinfo = self.network_api.allocate_for_instance( [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] created_port_ids = self._update_ports_for_instance( [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] with excutils.save_and_reraise_exception(): [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] self.force_reraise() [ 2249.549159] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] raise self.value [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] updated_port = self._update_port( [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] _ensure_no_port_binding_failure(port) [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] raise exception.PortBindingFailed(port_id=port['id']) [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] nova.exception.PortBindingFailed: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. [ 2249.549638] env[62506]: ERROR nova.compute.manager [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] [ 2249.549638] env[62506]: DEBUG nova.compute.utils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2249.550187] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.018s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2249.553585] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Build of instance df6be48d-bb76-4efd-9da5-3e6ce6467667 was re-scheduled: Binding failed for port 103c9a54-4f65-4ca2-b50f-329c69873c46, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2249.554145] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2249.554385] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2249.554566] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2249.554691] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2250.076482] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2250.124248] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85e43670-0bc1-4c9f-8036-e7a21ff3e99c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2250.131868] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f91948da-ec26-4c4f-bc11-4bc70dc57442 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2250.163520] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2250.165548] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ffa850e-65fb-4cc8-b0c3-7765b8c0a491 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2250.174237] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19bba981-de87-42fd-a81e-0e9f63896661 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2250.187741] env[62506]: DEBUG nova.compute.provider_tree [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2250.459120] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2250.563334] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "af5fbeea-eb1c-44af-ab9d-612430a0e5ee" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2250.563631] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "af5fbeea-eb1c-44af-ab9d-612430a0e5ee" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2250.669439] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-df6be48d-bb76-4efd-9da5-3e6ce6467667" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2250.669713] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2250.669906] env[62506]: DEBUG nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2250.670093] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2250.684706] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2250.689992] env[62506]: DEBUG nova.scheduler.client.report [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2251.029767] env[62506]: DEBUG nova.network.neutron [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2251.065760] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2251.187567] env[62506]: DEBUG nova.network.neutron [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2251.194855] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.645s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2251.195530] env[62506]: ERROR nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Traceback (most recent call last): [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self.driver.spawn(context, instance, image_meta, [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] vm_ref = self.build_virtual_machine(instance, [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] vif_infos = vmwarevif.get_vif_info(self._session, [ 2251.195530] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] for vif in network_info: [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] return self._sync_wrapper(fn, *args, **kwargs) [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self.wait() [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self[:] = self._gt.wait() [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] return self._exit_event.wait() [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] current.throw(*self._exc) [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2251.195903] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] result = function(*args, **kwargs) [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] return func(*args, **kwargs) [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] raise e [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] nwinfo = self.network_api.allocate_for_instance( [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] created_port_ids = self._update_ports_for_instance( [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] with excutils.save_and_reraise_exception(): [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] self.force_reraise() [ 2251.196396] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] raise self.value [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] updated_port = self._update_port( [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] _ensure_no_port_binding_failure(port) [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] raise exception.PortBindingFailed(port_id=port['id']) [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] nova.exception.PortBindingFailed: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. [ 2251.196752] env[62506]: ERROR nova.compute.manager [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] [ 2251.196752] env[62506]: DEBUG nova.compute.utils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2251.197897] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 8.362s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2251.198090] env[62506]: DEBUG nova.objects.instance [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62506) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 2251.200464] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Build of instance 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0 was re-scheduled: Binding failed for port ed6f4894-46f0-4939-a3c2-e756794f1b5b, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2251.200883] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2251.201124] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2251.201285] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2251.201447] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2251.532756] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Releasing lock "refresh_cache-e6f9199d-16c7-4d18-87a4-723be0eb3a2d" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2251.532942] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] [instance: e6f9199d-16c7-4d18-87a4-723be0eb3a2d] Updated the network info_cache for instance {{(pid=62506) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10516}} [ 2251.533176] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2251.533409] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2251.533613] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2251.533803] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2251.533983] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2251.534132] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 2251.586741] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2251.690060] env[62506]: INFO nova.compute.manager [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: df6be48d-bb76-4efd-9da5-3e6ce6467667] Took 1.02 seconds to deallocate network for instance. [ 2251.720642] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2251.792544] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2252.210284] env[62506]: DEBUG oslo_concurrency.lockutils [None req-ac883515-f6d8-4ca0-9439-b3ffae35453c tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.012s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2252.210723] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.835s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2252.294796] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2252.295035] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2252.295194] env[62506]: DEBUG nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2252.295361] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2252.310914] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2252.720755] env[62506]: INFO nova.scheduler.client.report [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Deleted allocations for instance df6be48d-bb76-4efd-9da5-3e6ce6467667 [ 2252.782555] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-846b550a-0e12-4676-a043-9e0d0266734d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2252.790217] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32133a8e-e7a6-429c-9178-3e68ecdf80c8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2252.819102] env[62506]: DEBUG nova.network.neutron [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2252.820731] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3200a288-d91f-40e5-9265-bcfc7e89dd6f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2252.828038] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66c10011-438f-475a-aeb2-efcb89ce651a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2252.841715] env[62506]: DEBUG nova.compute.provider_tree [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2253.232359] env[62506]: DEBUG oslo_concurrency.lockutils [None req-dea2e543-856d-4b3d-9b09-6b8850db78e6 tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "df6be48d-bb76-4efd-9da5-3e6ce6467667" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.541s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2253.324774] env[62506]: INFO nova.compute.manager [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0] Took 1.03 seconds to deallocate network for instance. [ 2253.344950] env[62506]: DEBUG nova.scheduler.client.report [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2253.849072] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.638s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2253.849734] env[62506]: ERROR nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Traceback (most recent call last): [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self.driver.spawn(context, instance, image_meta, [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] vm_ref = self.build_virtual_machine(instance, [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] vif_infos = vmwarevif.get_vif_info(self._session, [ 2253.849734] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] for vif in network_info: [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] return self._sync_wrapper(fn, *args, **kwargs) [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self.wait() [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self[:] = self._gt.wait() [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] return self._exit_event.wait() [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] current.throw(*self._exc) [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2253.850154] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] result = function(*args, **kwargs) [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] return func(*args, **kwargs) [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] raise e [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] nwinfo = self.network_api.allocate_for_instance( [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] created_port_ids = self._update_ports_for_instance( [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] with excutils.save_and_reraise_exception(): [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] self.force_reraise() [ 2253.850588] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] raise self.value [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] updated_port = self._update_port( [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] _ensure_no_port_binding_failure(port) [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] raise exception.PortBindingFailed(port_id=port['id']) [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] nova.exception.PortBindingFailed: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. [ 2253.851017] env[62506]: ERROR nova.compute.manager [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] [ 2253.851017] env[62506]: DEBUG nova.compute.utils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2253.851587] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 6.060s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2253.851818] env[62506]: DEBUG nova.objects.instance [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lazy-loading 'resources' on Instance uuid e6f9199d-16c7-4d18-87a4-723be0eb3a2d {{(pid=62506) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 2253.852901] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Build of instance 54ea1f68-8c18-4564-9743-6aec7d1995e9 was re-scheduled: Binding failed for port 9ae8ed80-25f5-4d9e-8212-eba4ab116a94, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2253.853333] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2253.853560] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquiring lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2253.853708] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Acquired lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2253.853956] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2254.230300] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "ebf06015-f3ef-4e59-93a1-2368d1666e86" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2254.230546] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "ebf06015-f3ef-4e59-93a1-2368d1666e86" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2254.355735] env[62506]: INFO nova.scheduler.client.report [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Deleted allocations for instance 0da31e2f-e2aa-4499-98cb-ff3d5a8745b0 [ 2254.382120] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2254.438502] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6596b37-ccba-4fa0-873c-9d7a8e418b08 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2254.446475] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cf0c7c2-2b2f-4eb6-b071-0c427ed26ec3 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2254.475695] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2254.477276] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4fabb30-4825-4f87-aace-52d1b060fcfc {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2254.484437] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-174db68f-422e-4b25-b24c-89fa6dd4a22a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2254.499846] env[62506]: DEBUG nova.compute.provider_tree [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2254.733381] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2254.867582] env[62506]: DEBUG oslo_concurrency.lockutils [None req-fa3798bc-f5a6-4be8-b500-b4817b317234 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "0da31e2f-e2aa-4499-98cb-ff3d5a8745b0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.447s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2254.981051] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Releasing lock "refresh_cache-54ea1f68-8c18-4564-9743-6aec7d1995e9" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2254.981319] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2254.981510] env[62506]: DEBUG nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2254.981677] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2254.996566] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2255.002900] env[62506]: DEBUG nova.scheduler.client.report [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2255.253450] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2255.500028] env[62506]: DEBUG nova.network.neutron [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2255.507710] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.656s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2255.510316] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.954s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2255.523449] env[62506]: INFO nova.scheduler.client.report [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Deleted allocations for instance e6f9199d-16c7-4d18-87a4-723be0eb3a2d [ 2255.991724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "3415368b-9d96-4c3e-baf2-aabfbe9861ad" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2255.992023] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "3415368b-9d96-4c3e-baf2-aabfbe9861ad" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2256.001585] env[62506]: INFO nova.compute.manager [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] [instance: 54ea1f68-8c18-4564-9743-6aec7d1995e9] Took 1.02 seconds to deallocate network for instance. [ 2256.030202] env[62506]: DEBUG oslo_concurrency.lockutils [None req-40aab16f-0768-4559-9f95-2752a1a666d7 tempest-ServerShowV257Test-1080414610 tempest-ServerShowV257Test-1080414610-project-member] Lock "e6f9199d-16c7-4d18-87a4-723be0eb3a2d" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 12.512s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2256.086202] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf323dfb-41de-4abd-be3f-08e04b469458 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.094143] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b192617b-697e-40e2-9886-bcfdfd6e8a45 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.125366] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e64eeec-0ce8-4917-bea9-03568f058feb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.132871] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c59610e-26f5-4bb7-a7b1-71329add90a8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.146084] env[62506]: DEBUG nova.compute.provider_tree [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2256.494743] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2256.649719] env[62506]: DEBUG nova.scheduler.client.report [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2257.019305] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2257.033621] env[62506]: INFO nova.scheduler.client.report [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Deleted allocations for instance 54ea1f68-8c18-4564-9743-6aec7d1995e9 [ 2257.154983] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.645s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2257.155699] env[62506]: ERROR nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Traceback (most recent call last): [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self.driver.spawn(context, instance, image_meta, [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] vm_ref = self.build_virtual_machine(instance, [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] vif_infos = vmwarevif.get_vif_info(self._session, [ 2257.155699] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] for vif in network_info: [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] return self._sync_wrapper(fn, *args, **kwargs) [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self.wait() [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self[:] = self._gt.wait() [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] return self._exit_event.wait() [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] current.throw(*self._exc) [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2257.156098] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] result = function(*args, **kwargs) [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] return func(*args, **kwargs) [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] raise e [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] nwinfo = self.network_api.allocate_for_instance( [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] created_port_ids = self._update_ports_for_instance( [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] with excutils.save_and_reraise_exception(): [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] self.force_reraise() [ 2257.156517] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] raise self.value [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] updated_port = self._update_port( [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] _ensure_no_port_binding_failure(port) [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] raise exception.PortBindingFailed(port_id=port['id']) [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] nova.exception.PortBindingFailed: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. [ 2257.156903] env[62506]: ERROR nova.compute.manager [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] [ 2257.156903] env[62506]: DEBUG nova.compute.utils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2257.157700] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 5.571s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2257.159892] env[62506]: INFO nova.compute.claims [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2257.162820] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Build of instance 2153994f-947b-47a5-8eca-f665c28eb87f was re-scheduled: Binding failed for port 767641d2-8f80-42ab-bd32-3820ef307258, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2257.163931] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2257.163931] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquiring lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2257.163931] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Acquired lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2257.163931] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2257.542676] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3d15d935-16ce-43d4-91ae-c98f4560181c tempest-ServerActionsTestOtherA-549345189 tempest-ServerActionsTestOtherA-549345189-project-member] Lock "54ea1f68-8c18-4564-9743-6aec7d1995e9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 27.151s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2257.710579] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2257.799870] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2258.254057] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-061140f9-97da-4f7b-a3f3-ca877bca368d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2258.262027] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d222b1b3-2a8b-40ae-af5d-4862619eacf5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2258.293372] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60d600e8-c272-4a7b-99eb-d043c30230e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2258.302352] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00780e8c-5bdc-4af4-a4a3-efad673864e1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2258.308105] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Releasing lock "refresh_cache-2153994f-947b-47a5-8eca-f665c28eb87f" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2258.309138] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2258.309138] env[62506]: DEBUG nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2258.309138] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2258.320937] env[62506]: DEBUG nova.compute.provider_tree [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2258.337871] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2258.827125] env[62506]: DEBUG nova.scheduler.client.report [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2258.840211] env[62506]: DEBUG nova.network.neutron [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2259.336028] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.175s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2259.336028] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2259.338702] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.085s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2259.340392] env[62506]: INFO nova.compute.claims [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2259.343957] env[62506]: INFO nova.compute.manager [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] [instance: 2153994f-947b-47a5-8eca-f665c28eb87f] Took 1.04 seconds to deallocate network for instance. [ 2259.535902] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquiring lock "1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2259.535902] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Lock "1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2259.840235] env[62506]: DEBUG nova.compute.utils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2259.841633] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2259.841924] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2259.930913] env[62506]: DEBUG nova.policy [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bbccae76c293437b84873cc656a055c3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a983195f14a44160a2cf5c170bd79ffd', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2260.038158] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Starting instance... {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 2260.284140] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Successfully created port: ad1c3c36-0f01-46ee-b7cc-3d156f880b78 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2260.353342] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2260.404643] env[62506]: INFO nova.scheduler.client.report [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Deleted allocations for instance 2153994f-947b-47a5-8eca-f665c28eb87f [ 2260.472186] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39f2df31-1af6-452f-9320-6ee4d4ca9437 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.480770] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a888a04a-56a4-4018-8373-0353bf5d3aea {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.513610] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef6f0856-b523-46b9-9b59-ae430f676012 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.521065] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83de8012-516e-459c-8719-b8369bc052ce {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.534020] env[62506]: DEBUG nova.compute.provider_tree [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2260.557345] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2260.919126] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9ffb89e9-0860-462c-90ba-6fec54d793d5 tempest-ServerMetadataNegativeTestJSON-388144728 tempest-ServerMetadataNegativeTestJSON-388144728-project-member] Lock "2153994f-947b-47a5-8eca-f665c28eb87f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 26.973s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2261.008966] env[62506]: DEBUG nova.compute.manager [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Received event network-changed-ad1c3c36-0f01-46ee-b7cc-3d156f880b78 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2261.009186] env[62506]: DEBUG nova.compute.manager [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Refreshing instance network info cache due to event network-changed-ad1c3c36-0f01-46ee-b7cc-3d156f880b78. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2261.009401] env[62506]: DEBUG oslo_concurrency.lockutils [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] Acquiring lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2261.009542] env[62506]: DEBUG oslo_concurrency.lockutils [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] Acquired lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2261.009697] env[62506]: DEBUG nova.network.neutron [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Refreshing network info cache for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2261.037513] env[62506]: DEBUG nova.scheduler.client.report [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2261.251737] env[62506]: ERROR nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2261.251737] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2261.251737] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2261.251737] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2261.251737] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2261.251737] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2261.251737] env[62506]: ERROR nova.compute.manager raise self.value [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2261.251737] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2261.251737] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2261.251737] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2261.252371] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2261.252371] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2261.252371] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2261.252371] env[62506]: ERROR nova.compute.manager [ 2261.252371] env[62506]: Traceback (most recent call last): [ 2261.252371] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2261.252371] env[62506]: listener.cb(fileno) [ 2261.252371] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2261.252371] env[62506]: result = function(*args, **kwargs) [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2261.252371] env[62506]: return func(*args, **kwargs) [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2261.252371] env[62506]: raise e [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2261.252371] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2261.252371] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2261.252371] env[62506]: with excutils.save_and_reraise_exception(): [ 2261.252371] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2261.252371] env[62506]: self.force_reraise() [ 2261.252371] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2261.252371] env[62506]: raise self.value [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2261.252371] env[62506]: updated_port = self._update_port( [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2261.252371] env[62506]: _ensure_no_port_binding_failure(port) [ 2261.252371] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2261.252371] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2261.253340] env[62506]: nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2261.253340] env[62506]: Removing descriptor: 15 [ 2261.364919] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2261.389079] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2261.389328] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2261.389486] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2261.389667] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2261.389811] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2261.389957] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2261.390185] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2261.390346] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2261.390508] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2261.390666] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2261.390856] env[62506]: DEBUG nova.virt.hardware [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2261.391731] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5254d98c-98c8-4481-b64d-d11dcd22ddcb {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.399660] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af70cfea-1d32-44df-93e6-3fa43a2395b6 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.413146] env[62506]: ERROR nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Traceback (most recent call last): [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] yield resources [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self.driver.spawn(context, instance, image_meta, [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] vm_ref = self.build_virtual_machine(instance, [ 2261.413146] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] vif_infos = vmwarevif.get_vif_info(self._session, [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] for vif in network_info: [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] return self._sync_wrapper(fn, *args, **kwargs) [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self.wait() [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self[:] = self._gt.wait() [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] return self._exit_event.wait() [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2261.413611] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] current.throw(*self._exc) [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] result = function(*args, **kwargs) [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] return func(*args, **kwargs) [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] raise e [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] nwinfo = self.network_api.allocate_for_instance( [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] created_port_ids = self._update_ports_for_instance( [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] with excutils.save_and_reraise_exception(): [ 2261.414107] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self.force_reraise() [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] raise self.value [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] updated_port = self._update_port( [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] _ensure_no_port_binding_failure(port) [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] raise exception.PortBindingFailed(port_id=port['id']) [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2261.414537] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] [ 2261.414537] env[62506]: INFO nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Terminating instance [ 2261.532055] env[62506]: DEBUG nova.network.neutron [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2261.541540] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2261.542023] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2261.544948] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.526s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2261.546210] env[62506]: INFO nova.compute.claims [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2261.659720] env[62506]: DEBUG nova.network.neutron [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2261.918253] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2262.049969] env[62506]: DEBUG nova.compute.utils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2262.053526] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2262.053706] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2262.145349] env[62506]: DEBUG nova.policy [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3c9c372663b34fcaa0dac3e1c657e044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fc10dc6a19fb41cdbf046f6da6dc69fe', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2262.164473] env[62506]: DEBUG oslo_concurrency.lockutils [req-6b12f3bc-6e55-4b6f-b38b-29d066b4bbfa req-e9d54400-f0e4-4000-9472-7a9f23863cc3 service nova] Releasing lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2262.164473] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquired lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2262.164473] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2262.440492] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Successfully created port: 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2262.554260] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2262.638280] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0a34244-9384-4276-9783-e00d42a37487 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.646451] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c86bab41-cec1-401d-8447-91eef354b526 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.678856] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cefcfb6a-4c57-4b6c-a9f8-9cfaf3c01980 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.686602] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67368821-f292-424e-aaab-82e1fdf59187 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.699480] env[62506]: DEBUG nova.compute.provider_tree [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2262.701526] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2262.839419] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2263.047403] env[62506]: DEBUG nova.compute.manager [req-20e8346e-80dd-415d-8e3d-fe8dceab7a2c req-b027ecad-88b5-4b49-9cfc-9194c6d1072b service nova] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Received event network-vif-deleted-ad1c3c36-0f01-46ee-b7cc-3d156f880b78 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2263.205056] env[62506]: DEBUG nova.scheduler.client.report [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2263.342217] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Releasing lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2263.343013] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2263.343013] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2263.343191] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f4e6f1f1-14b4-454a-b2d1-caebe321448c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.352885] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37e50c59-558f-4671-9451-c352b0934a55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.377395] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance af5fbeea-eb1c-44af-ab9d-612430a0e5ee could not be found. [ 2263.378047] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2263.378047] env[62506]: INFO nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2263.378246] env[62506]: DEBUG oslo.service.loopingcall [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2263.378556] env[62506]: DEBUG nova.compute.manager [-] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2263.378720] env[62506]: DEBUG nova.network.neutron [-] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2263.395483] env[62506]: DEBUG nova.network.neutron [-] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2263.425190] env[62506]: ERROR nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2263.425190] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2263.425190] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2263.425190] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2263.425190] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2263.425190] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2263.425190] env[62506]: ERROR nova.compute.manager raise self.value [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2263.425190] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2263.425190] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2263.425190] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2263.425758] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2263.425758] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2263.425758] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2263.425758] env[62506]: ERROR nova.compute.manager [ 2263.425758] env[62506]: Traceback (most recent call last): [ 2263.425758] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2263.425758] env[62506]: listener.cb(fileno) [ 2263.425758] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2263.425758] env[62506]: result = function(*args, **kwargs) [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2263.425758] env[62506]: return func(*args, **kwargs) [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2263.425758] env[62506]: raise e [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2263.425758] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2263.425758] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2263.425758] env[62506]: with excutils.save_and_reraise_exception(): [ 2263.425758] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2263.425758] env[62506]: self.force_reraise() [ 2263.425758] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2263.425758] env[62506]: raise self.value [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2263.425758] env[62506]: updated_port = self._update_port( [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2263.425758] env[62506]: _ensure_no_port_binding_failure(port) [ 2263.425758] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2263.425758] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2263.426734] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2263.426734] env[62506]: Removing descriptor: 15 [ 2263.567071] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2263.591705] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2263.591963] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2263.592139] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2263.592322] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2263.592465] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2263.592609] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2263.592816] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2263.592973] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2263.593161] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2263.593322] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2263.593490] env[62506]: DEBUG nova.virt.hardware [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2263.594442] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fac1c59a-2fa0-4237-a2f5-e525d122a81a {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.601501] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a57b151-c4cc-4cb5-bd33-c2c8b68f4d8f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.616374] env[62506]: ERROR nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Traceback (most recent call last): [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] yield resources [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self.driver.spawn(context, instance, image_meta, [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] vm_ref = self.build_virtual_machine(instance, [ 2263.616374] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] vif_infos = vmwarevif.get_vif_info(self._session, [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] for vif in network_info: [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] return self._sync_wrapper(fn, *args, **kwargs) [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self.wait() [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self[:] = self._gt.wait() [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] return self._exit_event.wait() [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2263.616799] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] current.throw(*self._exc) [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] result = function(*args, **kwargs) [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] return func(*args, **kwargs) [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] raise e [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] nwinfo = self.network_api.allocate_for_instance( [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] created_port_ids = self._update_ports_for_instance( [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] with excutils.save_and_reraise_exception(): [ 2263.617403] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self.force_reraise() [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] raise self.value [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] updated_port = self._update_port( [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] _ensure_no_port_binding_failure(port) [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] raise exception.PortBindingFailed(port_id=port['id']) [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2263.617990] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] [ 2263.617990] env[62506]: INFO nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Terminating instance [ 2263.710252] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.165s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2263.710755] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2263.713310] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.156s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2263.714737] env[62506]: INFO nova.compute.claims [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2263.898059] env[62506]: DEBUG nova.network.neutron [-] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2264.120059] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2264.120552] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2264.120792] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2264.219166] env[62506]: DEBUG nova.compute.utils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2264.222559] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2264.222767] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2264.265351] env[62506]: DEBUG nova.policy [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e508d2263c0c47d5bccfb1e2f1409316', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7d771ac74ac54fb38a84337eed72348a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2264.401599] env[62506]: INFO nova.compute.manager [-] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Took 1.02 seconds to deallocate network for instance. [ 2264.405423] env[62506]: DEBUG nova.compute.claims [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2264.405423] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2264.542243] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Successfully created port: 604e9828-b7e5-4e6a-932d-34e1af0dea56 {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2264.642764] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2264.723359] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2264.774163] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2264.807850] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-408d75fd-8766-4257-87cc-eabf25b527e9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.815519] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef15aeff-491b-4eb6-b690-bae963a494fd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.845545] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3af8e288-caaf-49c2-be10-d3c2bed21c55 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.852708] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17a8c857-0e99-459c-b8f0-94ec1cbfc1dd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.865529] env[62506]: DEBUG nova.compute.provider_tree [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2265.071178] env[62506]: DEBUG nova.compute.manager [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Received event network-changed-9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2265.071382] env[62506]: DEBUG nova.compute.manager [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Refreshing instance network info cache due to event network-changed-9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2265.071573] env[62506]: DEBUG oslo_concurrency.lockutils [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] Acquiring lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2265.275582] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2265.275989] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2265.276214] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2265.276517] env[62506]: DEBUG oslo_concurrency.lockutils [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] Acquired lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2265.276691] env[62506]: DEBUG nova.network.neutron [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Refreshing network info cache for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2265.277669] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ee213546-64c3-4e71-85e5-334841b7273f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.287557] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08a822e4-13dc-45a4-8ff2-7909b6b1950c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.315216] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ebf06015-f3ef-4e59-93a1-2368d1666e86 could not be found. [ 2265.315445] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2265.315627] env[62506]: INFO nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2265.315864] env[62506]: DEBUG oslo.service.loopingcall [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2265.316338] env[62506]: DEBUG nova.compute.manager [-] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2265.316442] env[62506]: DEBUG nova.network.neutron [-] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2265.342594] env[62506]: DEBUG nova.network.neutron [-] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2265.368709] env[62506]: DEBUG nova.scheduler.client.report [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2265.424747] env[62506]: ERROR nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2265.424747] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2265.424747] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2265.424747] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2265.424747] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2265.424747] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2265.424747] env[62506]: ERROR nova.compute.manager raise self.value [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2265.424747] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2265.424747] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2265.424747] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2265.425341] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2265.425341] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2265.425341] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2265.425341] env[62506]: ERROR nova.compute.manager [ 2265.425341] env[62506]: Traceback (most recent call last): [ 2265.425341] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2265.425341] env[62506]: listener.cb(fileno) [ 2265.425341] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2265.425341] env[62506]: result = function(*args, **kwargs) [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2265.425341] env[62506]: return func(*args, **kwargs) [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2265.425341] env[62506]: raise e [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2265.425341] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2265.425341] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2265.425341] env[62506]: with excutils.save_and_reraise_exception(): [ 2265.425341] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2265.425341] env[62506]: self.force_reraise() [ 2265.425341] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2265.425341] env[62506]: raise self.value [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2265.425341] env[62506]: updated_port = self._update_port( [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2265.425341] env[62506]: _ensure_no_port_binding_failure(port) [ 2265.425341] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2265.425341] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2265.426332] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2265.426332] env[62506]: Removing descriptor: 15 [ 2265.739402] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2265.775471] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2265.775719] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2265.775876] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2265.776075] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2265.776575] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2265.776575] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2265.776575] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2265.776745] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2265.776890] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2265.777052] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2265.777348] env[62506]: DEBUG nova.virt.hardware [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2265.778205] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b454980-b656-4491-a168-3655fd70f5e1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.787876] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e651320-9b42-4d47-8fb0-674d2e29787e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2265.803487] env[62506]: ERROR nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Traceback (most recent call last): [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] yield resources [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self.driver.spawn(context, instance, image_meta, [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] vm_ref = self.build_virtual_machine(instance, [ 2265.803487] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] vif_infos = vmwarevif.get_vif_info(self._session, [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] for vif in network_info: [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] return self._sync_wrapper(fn, *args, **kwargs) [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self.wait() [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self[:] = self._gt.wait() [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] return self._exit_event.wait() [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2265.804031] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] current.throw(*self._exc) [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] result = function(*args, **kwargs) [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] return func(*args, **kwargs) [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] raise e [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] nwinfo = self.network_api.allocate_for_instance( [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] created_port_ids = self._update_ports_for_instance( [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] with excutils.save_and_reraise_exception(): [ 2265.804527] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self.force_reraise() [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] raise self.value [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] updated_port = self._update_port( [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] _ensure_no_port_binding_failure(port) [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] raise exception.PortBindingFailed(port_id=port['id']) [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2265.804999] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] [ 2265.804999] env[62506]: INFO nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Terminating instance [ 2265.806497] env[62506]: DEBUG nova.network.neutron [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2265.844812] env[62506]: DEBUG nova.network.neutron [-] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2265.873091] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.160s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2265.873606] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Start building networks asynchronously for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2836}} [ 2265.876539] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 1.472s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2265.893990] env[62506]: DEBUG nova.network.neutron [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2266.311389] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2266.311580] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2266.311686] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2266.347106] env[62506]: INFO nova.compute.manager [-] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Took 1.03 seconds to deallocate network for instance. [ 2266.348999] env[62506]: DEBUG nova.compute.claims [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2266.349194] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2266.381183] env[62506]: DEBUG nova.compute.utils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Using /dev/sd instead of None {{(pid=62506) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2266.385390] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Allocating IP information in the background. {{(pid=62506) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1988}} [ 2266.385739] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] allocate_for_instance() {{(pid=62506) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 2266.396688] env[62506]: DEBUG oslo_concurrency.lockutils [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] Releasing lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2266.396688] env[62506]: DEBUG nova.compute.manager [req-d7050685-5538-43cd-b85a-f0811d8407b0 req-eab57cae-98ed-4b65-ae24-958ee30ca0d8 service nova] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Received event network-vif-deleted-9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2266.448041] env[62506]: DEBUG nova.policy [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9e62390799b24214ab1bb27173eb3a89', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd0833b6c62e1479e8fc92fb68651e8d8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62506) authorize /opt/stack/nova/nova/policy.py:192}} [ 2266.451093] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a62d9467-2e9f-4087-b6d9-53c8c3afd8d1 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2266.458220] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5b9f41c-8c5e-442e-842e-fb719fcec6cd {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2266.488105] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdea9cbd-ac06-4b61-b442-a2e8a96f5af9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2266.495315] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00579abb-bae5-4cd0-a805-3d365371fb82 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2266.508205] env[62506]: DEBUG nova.compute.provider_tree [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2266.731534] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Successfully created port: 1ab30809-76a0-44dc-8816-eb94cab2e9fd {{(pid=62506) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2266.844715] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2266.888008] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Start building block device mappings for instance. {{(pid=62506) _build_resources /opt/stack/nova/nova/compute/manager.py:2871}} [ 2266.975114] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2267.012125] env[62506]: DEBUG nova.scheduler.client.report [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2267.101403] env[62506]: DEBUG nova.compute.manager [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Received event network-changed-604e9828-b7e5-4e6a-932d-34e1af0dea56 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2267.101678] env[62506]: DEBUG nova.compute.manager [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Refreshing instance network info cache due to event network-changed-604e9828-b7e5-4e6a-932d-34e1af0dea56. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2267.101889] env[62506]: DEBUG oslo_concurrency.lockutils [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] Acquiring lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2267.477908] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2267.478433] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2267.478636] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2267.478950] env[62506]: DEBUG oslo_concurrency.lockutils [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] Acquired lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2267.479136] env[62506]: DEBUG nova.network.neutron [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Refreshing network info cache for port 604e9828-b7e5-4e6a-932d-34e1af0dea56 {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2267.480183] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e517b0ba-8ea7-47c6-8b06-afa21c13f3f9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2267.490443] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f3f1892-bee9-4b86-be41-3ef276f183a5 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2267.511906] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3415368b-9d96-4c3e-baf2-aabfbe9861ad could not be found. [ 2267.512159] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2267.512349] env[62506]: INFO nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2267.512608] env[62506]: DEBUG oslo.service.loopingcall [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2267.512797] env[62506]: DEBUG nova.compute.manager [-] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2267.512890] env[62506]: DEBUG nova.network.neutron [-] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2267.515180] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.639s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2267.515742] env[62506]: ERROR nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Traceback (most recent call last): [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self.driver.spawn(context, instance, image_meta, [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] vm_ref = self.build_virtual_machine(instance, [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] vif_infos = vmwarevif.get_vif_info(self._session, [ 2267.515742] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] for vif in network_info: [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] return self._sync_wrapper(fn, *args, **kwargs) [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self.wait() [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self[:] = self._gt.wait() [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] return self._exit_event.wait() [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] current.throw(*self._exc) [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2267.516114] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] result = function(*args, **kwargs) [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] return func(*args, **kwargs) [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] raise e [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] nwinfo = self.network_api.allocate_for_instance( [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] created_port_ids = self._update_ports_for_instance( [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] with excutils.save_and_reraise_exception(): [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] self.force_reraise() [ 2267.516640] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] raise self.value [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] updated_port = self._update_port( [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] _ensure_no_port_binding_failure(port) [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] raise exception.PortBindingFailed(port_id=port['id']) [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] nova.exception.PortBindingFailed: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. [ 2267.517047] env[62506]: ERROR nova.compute.manager [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] [ 2267.517047] env[62506]: DEBUG nova.compute.utils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2267.517699] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 1.168s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2267.521960] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Build of instance af5fbeea-eb1c-44af-ab9d-612430a0e5ee was re-scheduled: Binding failed for port ad1c3c36-0f01-46ee-b7cc-3d156f880b78, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2267.522398] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2267.522621] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquiring lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2267.522778] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Acquired lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2267.522920] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2267.539723] env[62506]: DEBUG nova.network.neutron [-] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2267.650731] env[62506]: ERROR nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2267.650731] env[62506]: ERROR nova.compute.manager Traceback (most recent call last): [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2267.650731] env[62506]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2267.650731] env[62506]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2267.650731] env[62506]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2267.650731] env[62506]: ERROR nova.compute.manager self.force_reraise() [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2267.650731] env[62506]: ERROR nova.compute.manager raise self.value [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2267.650731] env[62506]: ERROR nova.compute.manager updated_port = self._update_port( [ 2267.650731] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2267.650731] env[62506]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 2267.651533] env[62506]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2267.651533] env[62506]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 2267.651533] env[62506]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2267.651533] env[62506]: ERROR nova.compute.manager [ 2267.651533] env[62506]: Traceback (most recent call last): [ 2267.651533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 2267.651533] env[62506]: listener.cb(fileno) [ 2267.651533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2267.651533] env[62506]: result = function(*args, **kwargs) [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2267.651533] env[62506]: return func(*args, **kwargs) [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2267.651533] env[62506]: raise e [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2267.651533] env[62506]: nwinfo = self.network_api.allocate_for_instance( [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2267.651533] env[62506]: created_port_ids = self._update_ports_for_instance( [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2267.651533] env[62506]: with excutils.save_and_reraise_exception(): [ 2267.651533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2267.651533] env[62506]: self.force_reraise() [ 2267.651533] env[62506]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2267.651533] env[62506]: raise self.value [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2267.651533] env[62506]: updated_port = self._update_port( [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2267.651533] env[62506]: _ensure_no_port_binding_failure(port) [ 2267.651533] env[62506]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2267.651533] env[62506]: raise exception.PortBindingFailed(port_id=port['id']) [ 2267.652861] env[62506]: nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2267.652861] env[62506]: Removing descriptor: 15 [ 2267.899996] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Start spawning the instance on the hypervisor. {{(pid=62506) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2645}} [ 2267.924034] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-01-27T13:36:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-01-27T13:36:21Z,direct_url=,disk_format='vmdk',id=580d9529-f92b-46b3-a9d4-08013a8922b3,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='acd1926f4d5c403c816d31d1bfcc159c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-01-27T13:36:22Z,virtual_size=,visibility=), allow threads: False {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:567}} [ 2267.924313] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Flavor limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2267.924619] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Image limits 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:356}} [ 2267.924673] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Flavor pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2267.924780] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Image pref 0:0:0 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:396}} [ 2267.924930] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62506) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:434}} [ 2267.925307] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:573}} [ 2267.925410] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:475}} [ 2267.925580] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Got 1 possible topologies {{(pid=62506) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:505}} [ 2267.925742] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:579}} [ 2267.925915] env[62506]: DEBUG nova.virt.hardware [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62506) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:581}} [ 2267.926762] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bcfd6c8-e104-4913-9000-86ad302c4880 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2267.934684] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de6026cf-1e27-4bf4-aecb-f51907c9603c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2267.948478] env[62506]: ERROR nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Traceback (most recent call last): [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 2901, in _build_resources [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] yield resources [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self.driver.spawn(context, instance, image_meta, [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] vm_ref = self.build_virtual_machine(instance, [ 2267.948478] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] vif_infos = vmwarevif.get_vif_info(self._session, [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] for vif in network_info: [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] return self._sync_wrapper(fn, *args, **kwargs) [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self.wait() [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self[:] = self._gt.wait() [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] return self._exit_event.wait() [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2267.948979] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] current.throw(*self._exc) [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] result = function(*args, **kwargs) [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] return func(*args, **kwargs) [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] raise e [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] nwinfo = self.network_api.allocate_for_instance( [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] created_port_ids = self._update_ports_for_instance( [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] with excutils.save_and_reraise_exception(): [ 2267.949473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self.force_reraise() [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] raise self.value [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] updated_port = self._update_port( [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] _ensure_no_port_binding_failure(port) [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] raise exception.PortBindingFailed(port_id=port['id']) [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2267.949938] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] [ 2267.949938] env[62506]: INFO nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Terminating instance [ 2267.999872] env[62506]: DEBUG nova.network.neutron [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2268.041907] env[62506]: DEBUG nova.network.neutron [-] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2268.048836] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2268.088765] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e58ef66-e8fc-482b-91de-496422175272 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2268.096618] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b99b0651-eee9-483e-aeaf-7c30780d7864 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2268.129321] env[62506]: DEBUG nova.network.neutron [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2268.130820] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e20932b-ddb3-4176-90b0-d2584426c4b8 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2268.139601] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2168e67-3d8d-4823-825f-51036cf7bd35 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2268.154340] env[62506]: DEBUG nova.compute.provider_tree [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2268.183574] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2268.452716] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquiring lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2268.452891] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquired lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2268.453032] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2268.544400] env[62506]: INFO nova.compute.manager [-] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Took 1.03 seconds to deallocate network for instance. [ 2268.546547] env[62506]: DEBUG nova.compute.claims [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2268.546724] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2268.634901] env[62506]: DEBUG oslo_concurrency.lockutils [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] Releasing lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2268.635263] env[62506]: DEBUG nova.compute.manager [req-3b6f2605-4ad2-45d0-b71a-f48451591c70 req-7e693174-30eb-43cf-a3a3-62765d43d4d8 service nova] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Received event network-vif-deleted-604e9828-b7e5-4e6a-932d-34e1af0dea56 {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2268.657520] env[62506]: DEBUG nova.scheduler.client.report [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2268.685740] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Releasing lock "refresh_cache-af5fbeea-eb1c-44af-ab9d-612430a0e5ee" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2268.685961] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2268.686163] env[62506]: DEBUG nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2268.686347] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2268.700326] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2268.970856] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2269.037017] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2269.127915] env[62506]: DEBUG nova.compute.manager [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Received event network-changed-1ab30809-76a0-44dc-8816-eb94cab2e9fd {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2269.127915] env[62506]: DEBUG nova.compute.manager [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Refreshing instance network info cache due to event network-changed-1ab30809-76a0-44dc-8816-eb94cab2e9fd. {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11653}} [ 2269.127915] env[62506]: DEBUG oslo_concurrency.lockutils [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] Acquiring lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2269.162050] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.644s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2269.162640] env[62506]: ERROR nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Traceback (most recent call last): [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self.driver.spawn(context, instance, image_meta, [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] vm_ref = self.build_virtual_machine(instance, [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] vif_infos = vmwarevif.get_vif_info(self._session, [ 2269.162640] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] for vif in network_info: [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] return self._sync_wrapper(fn, *args, **kwargs) [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self.wait() [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self[:] = self._gt.wait() [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] return self._exit_event.wait() [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] current.throw(*self._exc) [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2269.163018] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] result = function(*args, **kwargs) [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] return func(*args, **kwargs) [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] raise e [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] nwinfo = self.network_api.allocate_for_instance( [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] created_port_ids = self._update_ports_for_instance( [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] with excutils.save_and_reraise_exception(): [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] self.force_reraise() [ 2269.163642] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] raise self.value [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] updated_port = self._update_port( [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] _ensure_no_port_binding_failure(port) [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] raise exception.PortBindingFailed(port_id=port['id']) [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] nova.exception.PortBindingFailed: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. [ 2269.164281] env[62506]: ERROR nova.compute.manager [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] [ 2269.164281] env[62506]: DEBUG nova.compute.utils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2269.164721] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.618s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2269.167533] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Build of instance ebf06015-f3ef-4e59-93a1-2368d1666e86 was re-scheduled: Binding failed for port 9a4cb9f4-6a55-41d2-81f4-a49fbd7facc2, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2269.167936] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2269.168173] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquiring lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2269.168324] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Acquired lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2269.168483] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2269.202331] env[62506]: DEBUG nova.network.neutron [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2269.539409] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Releasing lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2269.539838] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Start destroying the instance on the hypervisor. {{(pid=62506) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3164}} [ 2269.540064] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Destroying instance {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 2269.540452] env[62506]: DEBUG oslo_concurrency.lockutils [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] Acquired lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2269.540634] env[62506]: DEBUG nova.network.neutron [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Refreshing network info cache for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 2269.542043] env[62506]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bfd33b01-e91d-482e-8c52-f0f5c3c7f2ab {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2269.551324] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-682aacff-0e68-435a-a6d5-d5a725df2110 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2269.573204] env[62506]: WARNING nova.virt.vmwareapi.vmops [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae could not be found. [ 2269.573403] env[62506]: DEBUG nova.virt.vmwareapi.vmops [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance destroyed {{(pid=62506) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 2269.573575] env[62506]: INFO nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2269.573803] env[62506]: DEBUG oslo.service.loopingcall [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62506) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2269.574035] env[62506]: DEBUG nova.compute.manager [-] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2269.574137] env[62506]: DEBUG nova.network.neutron [-] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2269.588929] env[62506]: DEBUG nova.network.neutron [-] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2269.696597] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2269.704637] env[62506]: INFO nova.compute.manager [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] [instance: af5fbeea-eb1c-44af-ab9d-612430a0e5ee] Took 1.02 seconds to deallocate network for instance. [ 2269.734595] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccf1b960-c936-4243-b54f-ac09352bbbee {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2269.744576] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d59f31e-2344-423b-9050-ea5d5ae2be99 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2269.776618] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eceba596-b25d-4be7-9264-17017272d48c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2269.779423] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2269.785595] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3a79454-5634-48a0-aab5-dfd01ea86e57 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2269.800642] env[62506]: DEBUG nova.compute.provider_tree [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2270.060660] env[62506]: DEBUG nova.network.neutron [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2270.092559] env[62506]: DEBUG nova.network.neutron [-] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2270.127331] env[62506]: DEBUG nova.network.neutron [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2270.282184] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Releasing lock "refresh_cache-ebf06015-f3ef-4e59-93a1-2368d1666e86" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2270.282502] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2270.282708] env[62506]: DEBUG nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2270.282872] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2270.303185] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2270.304878] env[62506]: DEBUG nova.scheduler.client.report [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2270.597282] env[62506]: INFO nova.compute.manager [-] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Took 1.02 seconds to deallocate network for instance. [ 2270.599416] env[62506]: DEBUG nova.compute.claims [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Aborting claim: {{(pid=62506) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2270.599591] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2270.629331] env[62506]: DEBUG oslo_concurrency.lockutils [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] Releasing lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2270.629566] env[62506]: DEBUG nova.compute.manager [req-452113f8-bdea-4d6b-a2d5-d45e44e5dbea req-6d0c6a28-28e1-4904-a76a-15fc8e7ca2ba service nova] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Received event network-vif-deleted-1ab30809-76a0-44dc-8816-eb94cab2e9fd {{(pid=62506) external_instance_event /opt/stack/nova/nova/compute/manager.py:11648}} [ 2270.731233] env[62506]: INFO nova.scheduler.client.report [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Deleted allocations for instance af5fbeea-eb1c-44af-ab9d-612430a0e5ee [ 2270.809098] env[62506]: DEBUG nova.network.neutron [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2270.810638] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.646s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2270.811317] env[62506]: ERROR nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Traceback (most recent call last): [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self.driver.spawn(context, instance, image_meta, [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] vm_ref = self.build_virtual_machine(instance, [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] vif_infos = vmwarevif.get_vif_info(self._session, [ 2270.811317] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] for vif in network_info: [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] return self._sync_wrapper(fn, *args, **kwargs) [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self.wait() [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self[:] = self._gt.wait() [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] return self._exit_event.wait() [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] current.throw(*self._exc) [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2270.811736] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] result = function(*args, **kwargs) [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] return func(*args, **kwargs) [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] raise e [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] nwinfo = self.network_api.allocate_for_instance( [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] created_port_ids = self._update_ports_for_instance( [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] with excutils.save_and_reraise_exception(): [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] self.force_reraise() [ 2270.812196] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] raise self.value [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] updated_port = self._update_port( [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] _ensure_no_port_binding_failure(port) [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] raise exception.PortBindingFailed(port_id=port['id']) [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] nova.exception.PortBindingFailed: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. [ 2270.812594] env[62506]: ERROR nova.compute.manager [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] [ 2270.812594] env[62506]: DEBUG nova.compute.utils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2270.813160] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.214s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2270.816540] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Build of instance 3415368b-9d96-4c3e-baf2-aabfbe9861ad was re-scheduled: Binding failed for port 604e9828-b7e5-4e6a-932d-34e1af0dea56, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2270.816948] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2270.817246] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquiring lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2270.817407] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Acquired lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2270.817565] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2271.239538] env[62506]: DEBUG oslo_concurrency.lockutils [None req-51b17de3-11bf-4f02-b46e-d662d9ec4e17 tempest-AttachVolumeNegativeTest-87931072 tempest-AttachVolumeNegativeTest-87931072-project-member] Lock "af5fbeea-eb1c-44af-ab9d-612430a0e5ee" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 20.676s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2271.314435] env[62506]: INFO nova.compute.manager [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] [instance: ebf06015-f3ef-4e59-93a1-2368d1666e86] Took 1.03 seconds to deallocate network for instance. [ 2271.339012] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2271.378381] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e34a526-cefc-4e0c-8624-c3920d4e7e0e {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2271.385854] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c75e99c6-973f-4c21-bc64-354e4fb03185 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2271.418549] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-957fb8e6-fc6e-4b63-a6de-7d8fdad27535 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2271.425717] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2271.427721] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-959d7059-b5ed-4ca8-a5bc-9ae40f8e938c {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2271.442111] env[62506]: DEBUG nova.compute.provider_tree [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2271.933086] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Releasing lock "refresh_cache-3415368b-9d96-4c3e-baf2-aabfbe9861ad" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2271.933435] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2271.933635] env[62506]: DEBUG nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2271.933805] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2271.945017] env[62506]: DEBUG nova.scheduler.client.report [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2271.949850] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2272.346732] env[62506]: INFO nova.scheduler.client.report [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Deleted allocations for instance ebf06015-f3ef-4e59-93a1-2368d1666e86 [ 2272.451908] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.639s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2272.452631] env[62506]: ERROR nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Traceback (most recent call last): [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 2648, in _build_and_run_instance [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self.driver.spawn(context, instance, image_meta, [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 548, in spawn [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] vm_ref = self.build_virtual_machine(instance, [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] vif_infos = vmwarevif.get_vif_info(self._session, [ 2272.452631] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] for vif in network_info: [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/model.py", line 614, in __iter__ [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] return self._sync_wrapper(fn, *args, **kwargs) [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/model.py", line 605, in _sync_wrapper [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self.wait() [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/model.py", line 637, in wait [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self[:] = self._gt.wait() [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 232, in wait [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] return self._exit_event.wait() [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] current.throw(*self._exc) [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 272, in main [ 2272.453035] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] result = function(*args, **kwargs) [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] return func(*args, **kwargs) [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 2018, in _allocate_network_async [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] raise e [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/compute/manager.py", line 1996, in _allocate_network_async [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] nwinfo = self.network_api.allocate_for_instance( [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] created_port_ids = self._update_ports_for_instance( [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] with excutils.save_and_reraise_exception(): [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] self.force_reraise() [ 2272.453473] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] raise self.value [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] updated_port = self._update_port( [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] _ensure_no_port_binding_failure(port) [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] raise exception.PortBindingFailed(port_id=port['id']) [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] nova.exception.PortBindingFailed: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. [ 2272.453902] env[62506]: ERROR nova.compute.manager [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] [ 2272.453902] env[62506]: DEBUG nova.compute.utils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. {{(pid=62506) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2272.454591] env[62506]: DEBUG nova.network.neutron [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2272.455766] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Build of instance 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae was re-scheduled: Binding failed for port 1ab30809-76a0-44dc-8816-eb94cab2e9fd, please check neutron logs for more information. {{(pid=62506) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2487}} [ 2272.456192] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Unplugging VIFs for instance {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3013}} [ 2272.456422] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquiring lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2272.460019] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Acquired lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2272.460019] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Building network info cache for instance {{(pid=62506) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 2272.855399] env[62506]: DEBUG oslo_concurrency.lockutils [None req-3c336fe8-8318-4745-b293-2f31623e7aad tempest-DeleteServersTestJSON-1700850610 tempest-DeleteServersTestJSON-1700850610-project-member] Lock "ebf06015-f3ef-4e59-93a1-2368d1666e86" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 18.625s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2272.959702] env[62506]: INFO nova.compute.manager [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] [instance: 3415368b-9d96-4c3e-baf2-aabfbe9861ad] Took 1.03 seconds to deallocate network for instance. [ 2273.140256] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2273.268954] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2273.772244] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Releasing lock "refresh_cache-1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" {{(pid=62506) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2273.775138] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62506) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3036}} [ 2273.775138] env[62506]: DEBUG nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Deallocating network for instance {{(pid=62506) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2296}} [ 2273.775138] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] deallocate_for_instance() {{(pid=62506) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 2273.791757] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Instance cache missing network info. {{(pid=62506) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 2274.003018] env[62506]: INFO nova.scheduler.client.report [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Deleted allocations for instance 3415368b-9d96-4c3e-baf2-aabfbe9861ad [ 2274.294064] env[62506]: DEBUG nova.network.neutron [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Updating instance_info_cache with network_info: [] {{(pid=62506) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2274.509495] env[62506]: DEBUG oslo_concurrency.lockutils [None req-5b0458c0-854c-49bd-a775-ac55df0d9b98 tempest-ServersTestJSON-468355654 tempest-ServersTestJSON-468355654-project-member] Lock "3415368b-9d96-4c3e-baf2-aabfbe9861ad" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 18.517s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2274.796939] env[62506]: INFO nova.compute.manager [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] [instance: 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae] Took 1.02 seconds to deallocate network for instance. [ 2275.837379] env[62506]: INFO nova.scheduler.client.report [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Deleted allocations for instance 1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae [ 2276.348256] env[62506]: DEBUG oslo_concurrency.lockutils [None req-2b26df50-9a33-47c4-a1de-296564dd6ab8 tempest-ServersNegativeTestJSON-1739697766 tempest-ServersNegativeTestJSON-1739697766-project-member] Lock "1c12c5b5-9b45-4ff4-b24d-cd4dd7e3c1ae" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 16.812s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2284.611258] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2291.610800] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2293.113724] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager.update_available_resource {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2293.617085] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2293.617328] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2293.617519] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2293.617672] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62506) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2293.618625] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92222449-b007-4e7b-8f9d-0e805880e3e2 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2293.627569] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3889f60-be99-4726-a2e7-b6a83a803f7d {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2293.641654] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fe6aac8-06bc-4bf7-a0eb-322c18dd9d00 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2293.648070] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d34f2d66-3c71-40ff-8359-89c46885c170 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2293.677542] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181198MB free_disk=177GB free_vcpus=48 pci_devices=None {{(pid=62506) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2293.677662] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2293.677806] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2294.700240] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2294.700544] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62506) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2294.713833] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa5549a5-6928-4ada-8526-e46e0899e79f {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2294.721512] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-607fe677-675d-47ce-9799-80ac62a16b99 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2294.750375] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed009aad-89ca-4e6a-a70f-9890b27106a9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2294.757128] env[62506]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea056ff0-cfec-4e47-99c9-2b073d605aa9 {{(pid=62506) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2294.769581] env[62506]: DEBUG nova.compute.provider_tree [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed in ProviderTree for provider: 432b599d-3cbd-4439-9dff-9721e0cca9b4 {{(pid=62506) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2295.272175] env[62506]: DEBUG nova.scheduler.client.report [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Inventory has not changed for provider 432b599d-3cbd-4439-9dff-9721e0cca9b4 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 177, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62506) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:958}} [ 2295.778689] env[62506]: DEBUG nova.compute.resource_tracker [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62506) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2295.779159] env[62506]: DEBUG oslo_concurrency.lockutils [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.101s {{(pid=62506) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2296.276322] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2296.276596] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2296.276774] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2296.276951] env[62506]: DEBUG nova.compute.manager [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62506) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:11064}} [ 2296.610507] env[62506]: DEBUG oslo_service.periodic_task [None req-9722b5a9-f66d-40af-bc15-9b49b3d421a1 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62506) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}}